From 7c28259f92c2b3c608583ff128a5ae4134d4c48f Mon Sep 17 00:00:00 2001 From: mkaay Date: Wed, 30 Dec 2009 17:33:14 +0100 Subject: moved captcha stuff, extended serienjunkies, some other stuff --- module/Plugin.py | 5 +- module/captcha/GigasizeCom.py | 15 -- module/captcha/LinksaveIn.py | 161 ------------ module/captcha/LinksaveIn/bg/flecken_1.gif | Bin 10744 -> 0 bytes module/captcha/LinksaveIn/bg/flecken_2.gif | Bin 11076 -> 0 bytes module/captcha/LinksaveIn/bg/gewebe_fein.gif | Bin 10504 -> 0 bytes module/captcha/LinksaveIn/bg/gewebe_grob.gif | Bin 10127 -> 0 bytes module/captcha/LinksaveIn/bg/gitter.gif | Bin 8151 -> 0 bytes module/captcha/LinksaveIn/bg/mauer_horizontal.gif | Bin 9105 -> 0 bytes module/captcha/LinksaveIn/bg/mauer_vertikal.gif | Bin 10830 -> 0 bytes module/captcha/LinksaveIn/bg/scheckig.gif | Bin 10214 -> 0 bytes module/captcha/LinksaveIn/bg/wellen.gif | Bin 10041 -> 0 bytes module/captcha/LinksaveIn/tesser_conf | 1 - module/captcha/MegauploadCom.py | 13 - module/captcha/NetloadIn.py | 25 -- module/captcha/ShareonlineBiz.py | 61 ----- module/captcha/__init__.py | 0 module/captcha/captcha.py | 295 ---------------------- module/config/plugin_default.xml | 3 + module/download_thread.py | 14 +- module/file_list.py | 8 +- module/gui/CaptchaDock.py | 1 + module/plugins/captcha/GigasizeCom.py | 15 ++ module/plugins/captcha/LinksaveIn.py | 161 ++++++++++++ module/plugins/captcha/MegauploadCom.py | 13 + module/plugins/captcha/NetloadIn.py | 25 ++ module/plugins/captcha/ShareonlineBiz.py | 61 +++++ module/plugins/captcha/__init__.py | 0 module/plugins/captcha/captcha.py | 295 ++++++++++++++++++++++ module/plugins/decrypter/SerienjunkiesOrg.py | 178 ++++++++++++- module/thread_list.py | 23 +- 31 files changed, 776 insertions(+), 597 deletions(-) delete mode 100644 module/captcha/GigasizeCom.py delete mode 100644 module/captcha/LinksaveIn.py delete mode 100644 module/captcha/LinksaveIn/bg/flecken_1.gif delete mode 100644 module/captcha/LinksaveIn/bg/flecken_2.gif delete mode 100644 module/captcha/LinksaveIn/bg/gewebe_fein.gif delete mode 100644 module/captcha/LinksaveIn/bg/gewebe_grob.gif delete mode 100644 module/captcha/LinksaveIn/bg/gitter.gif delete mode 100644 module/captcha/LinksaveIn/bg/mauer_horizontal.gif delete mode 100644 module/captcha/LinksaveIn/bg/mauer_vertikal.gif delete mode 100644 module/captcha/LinksaveIn/bg/scheckig.gif delete mode 100644 module/captcha/LinksaveIn/bg/wellen.gif delete mode 100644 module/captcha/LinksaveIn/tesser_conf delete mode 100644 module/captcha/MegauploadCom.py delete mode 100644 module/captcha/NetloadIn.py delete mode 100644 module/captcha/ShareonlineBiz.py delete mode 100644 module/captcha/__init__.py delete mode 100644 module/captcha/captcha.py create mode 100644 module/plugins/captcha/GigasizeCom.py create mode 100644 module/plugins/captcha/LinksaveIn.py create mode 100644 module/plugins/captcha/MegauploadCom.py create mode 100644 module/plugins/captcha/NetloadIn.py create mode 100644 module/plugins/captcha/ShareonlineBiz.py create mode 100644 module/plugins/captcha/__init__.py create mode 100644 module/plugins/captcha/captcha.py (limited to 'module') diff --git a/module/Plugin.py b/module/Plugin.py index f3830595d..c33e0d565 100644 --- a/module/Plugin.py +++ b/module/Plugin.py @@ -32,7 +32,7 @@ class Plugin(): self.config = {} props = {} props['name'] = "BasePlugin" - props['version'] = "0.2" + props['version'] = "0.3" props['pattern'] = None props['type'] = "hoster" props['description'] = """Base Plugin""" @@ -47,6 +47,7 @@ class Plugin(): self.multi_dl = True self.ocr = None #captcha reader instance self.logger = logging.getLogger("log") + self.decryptNow = True def prepare(self, thread): pyfile = self.parent @@ -127,7 +128,7 @@ class Plugin(): pass def init_ocr(self): - modul = __import__("module.captcha." + self.props['name'], fromlist=['captcha']) + modul = __import__("module.plugins.captcha." + self.props['name'], fromlist=['captcha']) captchaClass = getattr(modul, self.props['name']) self.ocr = captchaClass() diff --git a/module/captcha/GigasizeCom.py b/module/captcha/GigasizeCom.py deleted file mode 100644 index 136092181..000000000 --- a/module/captcha/GigasizeCom.py +++ /dev/null @@ -1,15 +0,0 @@ -from captcha import OCR - -class GigasizeCom(OCR): - def __init__(self): - OCR.__init__(self) - - def get_captcha(self, image): - self.load_image(image) - self.threshold(2.8) - self.run_gocr() - return self.result_captcha - -if __name__ == '__main__': - ocr = GigasizeCom() - print ocr.get_captcha('gigasize-com/7.jpg') diff --git a/module/captcha/LinksaveIn.py b/module/captcha/LinksaveIn.py deleted file mode 100644 index d6f61e362..000000000 --- a/module/captcha/LinksaveIn.py +++ /dev/null @@ -1,161 +0,0 @@ -from captcha import OCR -import Image -from os import sep -from os.path import dirname -from os.path import abspath -from glob import glob -import tempfile - - -class LinksaveIn(OCR): - def __init__(self): - OCR.__init__(self) - self.data_dir = dirname(abspath(__file__)) + sep + "LinksaveIn" + sep - - def load_image(self, image): - im = Image.open(image) - frame_nr = 0 - - lut = im.resize((256, 1)) - lut.putdata(range(256)) - lut = list(lut.convert("RGB").getdata()) - - new = Image.new("RGB", im.size) - npix = new.load() - while True: - try: - im.seek(frame_nr) - except EOFError: - break - frame = im.copy() - pix = frame.load() - for x in range(frame.size[0]): - for y in range(frame.size[1]): - if lut[pix[x, y]] != (0,0,0): - npix[x, y] = lut[pix[x, y]] - frame_nr += 1 - new.save(self.data_dir+"unblacked.png") - self.image = new.copy() - self.pixels = self.image.load() - self.result_captcha = '' - - def get_bg(self): - stat = {} - cstat = {} - img = self.image.convert("P") - for bgpath in glob(self.data_dir+"bg/*.gif"): - stat[bgpath] = 0 - bg = Image.open(bgpath) - - bglut = bg.resize((256, 1)) - bglut.putdata(range(256)) - bglut = list(bglut.convert("RGB").getdata()) - - lut = img.resize((256, 1)) - lut.putdata(range(256)) - lut = list(lut.convert("RGB").getdata()) - - bgpix = bg.load() - pix = img.load() - for x in range(bg.size[0]): - for y in range(bg.size[1]): - rgb_bg = bglut[bgpix[x, y]] - rgb_c = lut[pix[x, y]] - try: - cstat[rgb_c] += 1 - except: - cstat[rgb_c] = 1 - if rgb_bg == rgb_c: - stat[bgpath] += 1 - max_p = 0 - bg = "" - for bgpath, value in stat.items(): - if max_p < value: - bg = bgpath - max_p = value - return bg - - def substract_bg(self, bgpath): - bg = Image.open(bgpath) - img = self.image.convert("P") - - bglut = bg.resize((256, 1)) - bglut.putdata(range(256)) - bglut = list(bglut.convert("RGB").getdata()) - - lut = img.resize((256, 1)) - lut.putdata(range(256)) - lut = list(lut.convert("RGB").getdata()) - - bgpix = bg.load() - pix = img.load() - orgpix = self.image.load() - for x in range(bg.size[0]): - for y in range(bg.size[1]): - rgb_bg = bglut[bgpix[x, y]] - rgb_c = lut[pix[x, y]] - if rgb_c == rgb_bg: - orgpix[x, y] = (255,255,255) - - def eval_black_white(self): - new = Image.new("RGB", (140, 75)) - pix = new.load() - orgpix = self.image.load() - thresh = 4 - for x in range(new.size[0]): - for y in range(new.size[1]): - rgb = orgpix[x, y] - r, g, b = rgb - pix[x, y] = (255,255,255) - if r > max(b, g)+thresh: - pix[x, y] = (0,0,0) - if g < min(r, b): - pix[x, y] = (0,0,0) - if g > max(r, b)+thresh: - pix[x, y] = (0,0,0) - if b > max(r, g)+thresh: - pix[x, y] = (0,0,0) - self.image = new - self.pixels = self.image.load() - - def run_tesser(self): - self.logger.debug("create tmp tif") - tmp = tempfile.NamedTemporaryFile(suffix=".tif") - self.logger.debug("create tmp txt") - tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") - self.logger.debug("save tiff") - self.image.save(tmp.name, 'TIFF') - self.logger.debug("run tesseract") - self.run(['tesseract', tmp.name, tmpTxt.name.replace(".txt", ""), "nobatch", self.data_dir+"tesser_conf"]) - self.logger.debug("read txt") - - with open(tmpTxt.name, 'r') as f: - self.result_captcha = f.read().replace("\n", "") - - def get_captcha(self, image): - self.load_image(image) - bg = self.get_bg() - self.substract_bg(bg) - self.eval_black_white() - self.to_greyscale() - self.image.save(self.data_dir+"cleaned_pass1.png") - self.clean(4) - self.clean(4) - self.image.save(self.data_dir+"cleaned_pass2.png") - letters = self.split_captcha_letters() - final = "" - for n, letter in enumerate(letters): - self.image = letter - self.image.save(ocr.data_dir+"letter%d.png" % n) - self.run_tesser() - final += self.result_captcha - - return final - -if __name__ == '__main__': - import urllib - ocr = LinksaveIn() - testurl = "http://linksave.in/captcha/cap.php?hsh=2229185&code=ZzHdhl3UffV3lXTH5U4b7nShXj%2Bwma1vyoNBcbc6lcc%3D" - urllib.urlretrieve(testurl, ocr.data_dir+"captcha.gif") - - print ocr.get_captcha(ocr.data_dir+'captcha.gif') diff --git a/module/captcha/LinksaveIn/bg/flecken_1.gif b/module/captcha/LinksaveIn/bg/flecken_1.gif deleted file mode 100644 index df2f51217..000000000 Binary files a/module/captcha/LinksaveIn/bg/flecken_1.gif and /dev/null differ diff --git a/module/captcha/LinksaveIn/bg/flecken_2.gif b/module/captcha/LinksaveIn/bg/flecken_2.gif deleted file mode 100644 index 838276188..000000000 Binary files a/module/captcha/LinksaveIn/bg/flecken_2.gif and /dev/null differ diff --git a/module/captcha/LinksaveIn/bg/gewebe_fein.gif b/module/captcha/LinksaveIn/bg/gewebe_fein.gif deleted file mode 100644 index 502f18cc4..000000000 Binary files a/module/captcha/LinksaveIn/bg/gewebe_fein.gif and /dev/null differ diff --git a/module/captcha/LinksaveIn/bg/gewebe_grob.gif b/module/captcha/LinksaveIn/bg/gewebe_grob.gif deleted file mode 100644 index e66a365ad..000000000 Binary files a/module/captcha/LinksaveIn/bg/gewebe_grob.gif and /dev/null differ diff --git a/module/captcha/LinksaveIn/bg/gitter.gif b/module/captcha/LinksaveIn/bg/gitter.gif deleted file mode 100644 index ec52ef68d..000000000 Binary files a/module/captcha/LinksaveIn/bg/gitter.gif and /dev/null differ diff --git a/module/captcha/LinksaveIn/bg/mauer_horizontal.gif b/module/captcha/LinksaveIn/bg/mauer_horizontal.gif deleted file mode 100644 index 3d75fafa8..000000000 Binary files a/module/captcha/LinksaveIn/bg/mauer_horizontal.gif and /dev/null differ diff --git a/module/captcha/LinksaveIn/bg/mauer_vertikal.gif b/module/captcha/LinksaveIn/bg/mauer_vertikal.gif deleted file mode 100644 index 2ada6fdae..000000000 Binary files a/module/captcha/LinksaveIn/bg/mauer_vertikal.gif and /dev/null differ diff --git a/module/captcha/LinksaveIn/bg/scheckig.gif b/module/captcha/LinksaveIn/bg/scheckig.gif deleted file mode 100644 index 8bfb45c56..000000000 Binary files a/module/captcha/LinksaveIn/bg/scheckig.gif and /dev/null differ diff --git a/module/captcha/LinksaveIn/bg/wellen.gif b/module/captcha/LinksaveIn/bg/wellen.gif deleted file mode 100644 index a181ebe74..000000000 Binary files a/module/captcha/LinksaveIn/bg/wellen.gif and /dev/null differ diff --git a/module/captcha/LinksaveIn/tesser_conf b/module/captcha/LinksaveIn/tesser_conf deleted file mode 100644 index 34ca8fa02..000000000 --- a/module/captcha/LinksaveIn/tesser_conf +++ /dev/null @@ -1 +0,0 @@ -tessedit_char_whitelist 0123456789 diff --git a/module/captcha/MegauploadCom.py b/module/captcha/MegauploadCom.py deleted file mode 100644 index 374bcd678..000000000 --- a/module/captcha/MegauploadCom.py +++ /dev/null @@ -1,13 +0,0 @@ -from captcha import OCR - -class MegauploadCom(OCR): - def __init__(self): - OCR.__init__(self) - - def get_captcha(self, image): - self.load_image(image) - self.run_tesser() - return self.result_captcha - -if __name__ == '__main__': - ocr = MegauploadCom() diff --git a/module/captcha/NetloadIn.py b/module/captcha/NetloadIn.py deleted file mode 100644 index 9799a6a2b..000000000 --- a/module/captcha/NetloadIn.py +++ /dev/null @@ -1,25 +0,0 @@ -from captcha import OCR - -class NetloadIn(OCR): - def __init__(self): - OCR.__init__(self) - - def get_captcha(self, image): - self.load_image(image) - self.to_greyscale() - self.clean(3) - self.clean(3) - self.run_tesser() - - self.correct({ - ("$", "g"): "5", - }) - - return self.result_captcha - -if __name__ == '__main__': - import urllib - ocr = NetloadIn() - urllib.urlretrieve("http://netload.in/share/includes/captcha.php", "captcha.png") - - print ocr.get_captcha('captcha.png') diff --git a/module/captcha/ShareonlineBiz.py b/module/captcha/ShareonlineBiz.py deleted file mode 100644 index 91124f181..000000000 --- a/module/captcha/ShareonlineBiz.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2009 kingzero, RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -### -from captcha import OCR - -class ShareonlineBiz(OCR): - def __init__(self): - OCR.__init__(self) - - def get_captcha(self, image): - self.load_image(image) - self.to_greyscale() - self.image = self.image.resize((160, 50)) - self.pixels = self.image.load() - self.threshold(1.85) - #self.eval_black_white(240) - #self.derotate_by_average() - - letters = self.split_captcha_letters() - - final = "" - for letter in letters: - self.image = letter - self.run_tesser() - final += self.result_captcha - - #replace common errors - final = self.correct({ - "A": "4", - "‘5": "3", - ("‘1", "T"): "7", - ("‘L", "B", "'L"): "2", - "b": "6", - ("I", "X"): "1" - }, final) - - return final - - #tesseract at 60% - -if __name__ == '__main__': - import urllib - ocr = ShareonlineBiz() - urllib.urlretrieve("http://www.share-online.biz/captcha.php", "captcha.jpeg") - print ocr.get_captcha('captcha.jpeg') diff --git a/module/captcha/__init__.py b/module/captcha/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/module/captcha/captcha.py b/module/captcha/captcha.py deleted file mode 100644 index 283b171e0..000000000 --- a/module/captcha/captcha.py +++ /dev/null @@ -1,295 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2009 kingzero, RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -### -from __future__ import with_statement -import logging -import subprocess -import tempfile -import threading - -import Image - -class RunThread(threading.Thread): - def __init__(self): - threading.Thread.__init__(self) - - def e(self, command, inputdata=None): - """execute command """ - self.command = command - self.inputdata = inputdata - self.result = "" - self.start() - self.join(10) - return self.result - - def run(self): - """Run a command and return standard output""" - pipe = subprocess.PIPE - popen = subprocess.Popen(self.command, stdout=pipe, stderr=pipe) - outputdata, errdata = popen.communicate(self.inputdata) - assert (popen.returncode == 0), \ - "Error running: %s\n\n%s" % (self.command, errdata) - self.result = outputdata - -class OCR(object): - def __init__(self): - self.logger = logging.getLogger("log") - - def load_image(self, image): - self.image = Image.open(image) - self.pixels = self.image.load() - self.result_captcha = '' - - def unload(self): - """delete all tmp images""" - pass - - def threshold(self, value): - self.image = self.image.point(lambda a: a * value + 10) - - def run(self, command, inputdata=None): - """Run a command and return standard output""" - # OLD METHOD - # pipe = subprocess.PIPE - # popen = subprocess.Popen(command, stdout=pipe, stderr=pipe) - # outputdata, errdata = popen.communicate(inputdata) - # assert (popen.returncode == 0), \ - # "Error running: %s\n\n%s" % (command, errdata) - # return outputdata - - thread = RunThread() - result = thread.e(command, inputdata) - return result - - def run_gocr(self): - tmp = tempfile.NamedTemporaryFile(suffix=".jpg") - self.image.save(tmp) - self.result_captcha = self.run(['gocr', tmp.name]).replace("\n", "") - - def run_tesser(self): - self.logger.debug("create tmp tif") - tmp = tempfile.NamedTemporaryFile(suffix=".tif") - self.logger.debug("create tmp txt") - tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") - self.logger.debug("save tiff") - self.image.save(tmp.name, 'TIFF') - self.logger.debug("run tesseract") - self.run(['tesseract', tmp.name, tmpTxt.name.replace(".txt", "")]) - self.logger.debug("read txt") - - with open(tmpTxt.name, 'r') as f: - self.result_captcha = f.read().replace("\n", "") - - def get_captcha(self): - raise NotImplementedError - - def to_greyscale(self): - if self.image.mode != 'L': - self.image = self.image.convert('L') - - self.pixels = self.image.load() - - def eval_black_white(self, limit): - self.pixels = self.image.load() - w, h = self.image.size - for x in xrange(w): - for y in xrange(h): - if self.pixels[x, y] > limit: - self.pixels[x, y] = 255 - else: - self.pixels[x, y] = 0 - - def clean(self, allowed): - pixels = self.pixels - - w, h = self.image.size - - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 255: continue - # no point in processing white pixels since we only want to remove black pixel - count = 0 - - try: - if pixels[x-1, y-1] != 255: count += 1 - if pixels[x-1, y] != 255: count += 1 - if pixels[x-1, y + 1] != 255: count += 1 - if pixels[x, y + 1] != 255: count += 1 - if pixels[x + 1, y + 1] != 255: count += 1 - if pixels[x + 1, y] != 255: count += 1 - if pixels[x + 1, y-1] != 255: count += 1 - if pixels[x, y-1] != 255: count += 1 - except: - pass - - # not enough neighbors are dark pixels so mark this pixel - # to be changed to white - if count < allowed: - pixels[x, y] = 1 - - # second pass: this time set all 1's to 255 (white) - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 1: pixels[x, y] = 255 - - self.pixels = pixels - - def derotate_by_average(self): - """rotate by checking each angle and guess most suitable""" - - w, h = self.image.size - pixels = self.pixels - - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 0: - pixels[x, y] = 155 - - highest = {} - counts = {} - - for angle in range(-45, 45): - - tmpimage = self.image.rotate(angle) - - pixels = tmpimage.load() - - w, h = self.image.size - - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 0: - pixels[x, y] = 255 - - - count = {} - - for x in xrange(w): - count[x] = 0 - for y in xrange(h): - if pixels[x, y] == 155: - count[x] += 1 - - sum = 0 - cnt = 0 - - for x in count.values(): - if x != 0: - sum += x - cnt += 1 - - avg = sum / cnt - counts[angle] = cnt - highest[angle] = 0 - for x in count.values(): - if x > highest[angle]: - highest[angle] = x - - highest[angle] = highest[angle] - avg - - hkey = 0 - hvalue = 0 - - for key, value in highest.iteritems(): - if value > hvalue: - hkey = key - hvalue = value - - self.image = self.image.rotate(hkey) - pixels = self.image.load() - - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 0: - pixels[x, y] = 255 - - if pixels[x, y] == 155: - pixels[x, y] = 0 - - self.pixels = pixels - - def split_captcha_letters(self): - captcha = self.image - started = False - letters = [] - width, height = captcha.size - bottomY, topY = 0, height - pixels = captcha.load() - - for x in xrange(width): - black_pixel_in_col = False - for y in xrange(height): - if pixels[x, y] != 255: - if started == False: - started = True - firstX = x - lastX = x - - if y > bottomY: bottomY = y - if y < topY: topY = y - if x > lastX: lastX = x - - black_pixel_in_col = True - - if black_pixel_in_col == False and started == True: - rect = (firstX, topY, lastX, bottomY) - new_captcha = captcha.crop(rect) - - w, h = new_captcha.size - if w > 5 and h > 5: - letters.append(new_captcha) - - started = False - bottomY, topY = 0, height - - return letters - - def correct(self, values, var=None): - - if var: - result = var - else: - result = self.result_captcha - - for key, item in values.iteritems(): - - if key.__class__ == str: - result = result.replace(key, item) - else: - for expr in key: - result = result.replace(expr, item) - - if var: - return result - else: - self.result_captcha = result - - -if __name__ == '__main__': - ocr = OCR() - ocr.load_image("B.jpg") - ocr.to_greyscale() - ocr.eval_black_white(140) - ocr.derotate_by_avergage() - ocr.run_gocr() - print "GOCR", ocr.result_captcha - ocr.run_tesser() - print "Tesseract", ocr.result_captcha - ocr.image.save("derotated.jpg") - diff --git a/module/config/plugin_default.xml b/module/config/plugin_default.xml index 01c0e7ed6..88628cb5e 100644 --- a/module/config/plugin_default.xml +++ b/module/config/plugin_default.xml @@ -36,4 +36,7 @@ False + + RapidshareCom,UploadedTo,NetloadIn,FilefactoryCom + diff --git a/module/download_thread.py b/module/download_thread.py index 3c008d000..a07f4511f 100644 --- a/module/download_thread.py +++ b/module/download_thread.py @@ -66,6 +66,9 @@ class Checksum(Exception): def getFile(self): return self.file +class CaptchaError(Exception): + pass + class Download_Thread(threading.Thread): def __init__(self, parent): threading.Thread.__init__(self) @@ -93,8 +96,10 @@ class Download_Thread(threading.Thread): f = open("%s.info" % e.getFile(), "w") f.write("Checksum not matched!") f.close() + except CaptchaError: + self.loadedPyFile.status.type = "failed" + self.loadedPyFile.status.error = "Can't solve captcha" except Exception, e: - try: if self.parent.parent.config['general']['debug_mode']: traceback.print_exc() @@ -123,8 +128,11 @@ class Download_Thread(threading.Thread): pyfile.plugin.prepare(self) pyfile.plugin.req.set_timeout(self.parent.parent.config['general']['max_download_time']) - - status.type = "downloading" + + if pyfile.plugin.props["type"] == "container": + status.type = "decrypting" + else: + status.type = "downloading" location = join(pyfile.folder, status.filename) pyfile.plugin.proceed(status.url, location) diff --git a/module/file_list.py b/module/file_list.py index cc3b63006..8af66d5ed 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -129,7 +129,11 @@ class File_List(object): files = [] for pypack in self.data["queue"] + self.data["packages"]: for pyfile in pypack.files: - if pyfile.plugin.props['type'] == "container" and not pyfile.active: + if pyfile.status.type == None and pyfile.plugin.props['type'] == "container" and not pyfile.active: + files.append(pyfile) + for pypack in self.data["packages"]: + for pyfile in pypack.files: + if pyfile.status.type == None and pyfile.plugin.props['type'] == "container" and pyfile.plugin.decryptNow and not pyfile.active: files.append(pyfile) for pypack in self.data["queue"]: for pyfile in pypack.files: @@ -424,7 +428,7 @@ class PyLoadFile(): for dir in ["hoster", "decrypter", "container"]: try: self.modul = __import__("%s.%s" % (dir, pluginName), globals(), locals(), [pluginName], -1) - except: + except Exception, e: pass pluginClass = getattr(self.modul, pluginName) else: diff --git a/module/gui/CaptchaDock.py b/module/gui/CaptchaDock.py index 3dc9441a4..8a7e8010e 100644 --- a/module/gui/CaptchaDock.py +++ b/module/gui/CaptchaDock.py @@ -41,6 +41,7 @@ class CaptchaDock(QDockWidget): data = QByteArray(img) self.currentID = tid self.widget.emit(SIGNAL("setImage"), data) + self.widget.input.setText("") self.show() class CaptchaDockWidget(QWidget): diff --git a/module/plugins/captcha/GigasizeCom.py b/module/plugins/captcha/GigasizeCom.py new file mode 100644 index 000000000..136092181 --- /dev/null +++ b/module/plugins/captcha/GigasizeCom.py @@ -0,0 +1,15 @@ +from captcha import OCR + +class GigasizeCom(OCR): + def __init__(self): + OCR.__init__(self) + + def get_captcha(self, image): + self.load_image(image) + self.threshold(2.8) + self.run_gocr() + return self.result_captcha + +if __name__ == '__main__': + ocr = GigasizeCom() + print ocr.get_captcha('gigasize-com/7.jpg') diff --git a/module/plugins/captcha/LinksaveIn.py b/module/plugins/captcha/LinksaveIn.py new file mode 100644 index 000000000..d6f61e362 --- /dev/null +++ b/module/plugins/captcha/LinksaveIn.py @@ -0,0 +1,161 @@ +from captcha import OCR +import Image +from os import sep +from os.path import dirname +from os.path import abspath +from glob import glob +import tempfile + + +class LinksaveIn(OCR): + def __init__(self): + OCR.__init__(self) + self.data_dir = dirname(abspath(__file__)) + sep + "LinksaveIn" + sep + + def load_image(self, image): + im = Image.open(image) + frame_nr = 0 + + lut = im.resize((256, 1)) + lut.putdata(range(256)) + lut = list(lut.convert("RGB").getdata()) + + new = Image.new("RGB", im.size) + npix = new.load() + while True: + try: + im.seek(frame_nr) + except EOFError: + break + frame = im.copy() + pix = frame.load() + for x in range(frame.size[0]): + for y in range(frame.size[1]): + if lut[pix[x, y]] != (0,0,0): + npix[x, y] = lut[pix[x, y]] + frame_nr += 1 + new.save(self.data_dir+"unblacked.png") + self.image = new.copy() + self.pixels = self.image.load() + self.result_captcha = '' + + def get_bg(self): + stat = {} + cstat = {} + img = self.image.convert("P") + for bgpath in glob(self.data_dir+"bg/*.gif"): + stat[bgpath] = 0 + bg = Image.open(bgpath) + + bglut = bg.resize((256, 1)) + bglut.putdata(range(256)) + bglut = list(bglut.convert("RGB").getdata()) + + lut = img.resize((256, 1)) + lut.putdata(range(256)) + lut = list(lut.convert("RGB").getdata()) + + bgpix = bg.load() + pix = img.load() + for x in range(bg.size[0]): + for y in range(bg.size[1]): + rgb_bg = bglut[bgpix[x, y]] + rgb_c = lut[pix[x, y]] + try: + cstat[rgb_c] += 1 + except: + cstat[rgb_c] = 1 + if rgb_bg == rgb_c: + stat[bgpath] += 1 + max_p = 0 + bg = "" + for bgpath, value in stat.items(): + if max_p < value: + bg = bgpath + max_p = value + return bg + + def substract_bg(self, bgpath): + bg = Image.open(bgpath) + img = self.image.convert("P") + + bglut = bg.resize((256, 1)) + bglut.putdata(range(256)) + bglut = list(bglut.convert("RGB").getdata()) + + lut = img.resize((256, 1)) + lut.putdata(range(256)) + lut = list(lut.convert("RGB").getdata()) + + bgpix = bg.load() + pix = img.load() + orgpix = self.image.load() + for x in range(bg.size[0]): + for y in range(bg.size[1]): + rgb_bg = bglut[bgpix[x, y]] + rgb_c = lut[pix[x, y]] + if rgb_c == rgb_bg: + orgpix[x, y] = (255,255,255) + + def eval_black_white(self): + new = Image.new("RGB", (140, 75)) + pix = new.load() + orgpix = self.image.load() + thresh = 4 + for x in range(new.size[0]): + for y in range(new.size[1]): + rgb = orgpix[x, y] + r, g, b = rgb + pix[x, y] = (255,255,255) + if r > max(b, g)+thresh: + pix[x, y] = (0,0,0) + if g < min(r, b): + pix[x, y] = (0,0,0) + if g > max(r, b)+thresh: + pix[x, y] = (0,0,0) + if b > max(r, g)+thresh: + pix[x, y] = (0,0,0) + self.image = new + self.pixels = self.image.load() + + def run_tesser(self): + self.logger.debug("create tmp tif") + tmp = tempfile.NamedTemporaryFile(suffix=".tif") + self.logger.debug("create tmp txt") + tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") + self.logger.debug("save tiff") + self.image.save(tmp.name, 'TIFF') + self.logger.debug("run tesseract") + self.run(['tesseract', tmp.name, tmpTxt.name.replace(".txt", ""), "nobatch", self.data_dir+"tesser_conf"]) + self.logger.debug("read txt") + + with open(tmpTxt.name, 'r') as f: + self.result_captcha = f.read().replace("\n", "") + + def get_captcha(self, image): + self.load_image(image) + bg = self.get_bg() + self.substract_bg(bg) + self.eval_black_white() + self.to_greyscale() + self.image.save(self.data_dir+"cleaned_pass1.png") + self.clean(4) + self.clean(4) + self.image.save(self.data_dir+"cleaned_pass2.png") + letters = self.split_captcha_letters() + final = "" + for n, letter in enumerate(letters): + self.image = letter + self.image.save(ocr.data_dir+"letter%d.png" % n) + self.run_tesser() + final += self.result_captcha + + return final + +if __name__ == '__main__': + import urllib + ocr = LinksaveIn() + testurl = "http://linksave.in/captcha/cap.php?hsh=2229185&code=ZzHdhl3UffV3lXTH5U4b7nShXj%2Bwma1vyoNBcbc6lcc%3D" + urllib.urlretrieve(testurl, ocr.data_dir+"captcha.gif") + + print ocr.get_captcha(ocr.data_dir+'captcha.gif') diff --git a/module/plugins/captcha/MegauploadCom.py b/module/plugins/captcha/MegauploadCom.py new file mode 100644 index 000000000..374bcd678 --- /dev/null +++ b/module/plugins/captcha/MegauploadCom.py @@ -0,0 +1,13 @@ +from captcha import OCR + +class MegauploadCom(OCR): + def __init__(self): + OCR.__init__(self) + + def get_captcha(self, image): + self.load_image(image) + self.run_tesser() + return self.result_captcha + +if __name__ == '__main__': + ocr = MegauploadCom() diff --git a/module/plugins/captcha/NetloadIn.py b/module/plugins/captcha/NetloadIn.py new file mode 100644 index 000000000..9799a6a2b --- /dev/null +++ b/module/plugins/captcha/NetloadIn.py @@ -0,0 +1,25 @@ +from captcha import OCR + +class NetloadIn(OCR): + def __init__(self): + OCR.__init__(self) + + def get_captcha(self, image): + self.load_image(image) + self.to_greyscale() + self.clean(3) + self.clean(3) + self.run_tesser() + + self.correct({ + ("$", "g"): "5", + }) + + return self.result_captcha + +if __name__ == '__main__': + import urllib + ocr = NetloadIn() + urllib.urlretrieve("http://netload.in/share/includes/captcha.php", "captcha.png") + + print ocr.get_captcha('captcha.png') diff --git a/module/plugins/captcha/ShareonlineBiz.py b/module/plugins/captcha/ShareonlineBiz.py new file mode 100644 index 000000000..91124f181 --- /dev/null +++ b/module/plugins/captcha/ShareonlineBiz.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2009 kingzero, RaNaN +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +### +from captcha import OCR + +class ShareonlineBiz(OCR): + def __init__(self): + OCR.__init__(self) + + def get_captcha(self, image): + self.load_image(image) + self.to_greyscale() + self.image = self.image.resize((160, 50)) + self.pixels = self.image.load() + self.threshold(1.85) + #self.eval_black_white(240) + #self.derotate_by_average() + + letters = self.split_captcha_letters() + + final = "" + for letter in letters: + self.image = letter + self.run_tesser() + final += self.result_captcha + + #replace common errors + final = self.correct({ + "A": "4", + "‘5": "3", + ("‘1", "T"): "7", + ("‘L", "B", "'L"): "2", + "b": "6", + ("I", "X"): "1" + }, final) + + return final + + #tesseract at 60% + +if __name__ == '__main__': + import urllib + ocr = ShareonlineBiz() + urllib.urlretrieve("http://www.share-online.biz/captcha.php", "captcha.jpeg") + print ocr.get_captcha('captcha.jpeg') diff --git a/module/plugins/captcha/__init__.py b/module/plugins/captcha/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/module/plugins/captcha/captcha.py b/module/plugins/captcha/captcha.py new file mode 100644 index 000000000..283b171e0 --- /dev/null +++ b/module/plugins/captcha/captcha.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2009 kingzero, RaNaN +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +### +from __future__ import with_statement +import logging +import subprocess +import tempfile +import threading + +import Image + +class RunThread(threading.Thread): + def __init__(self): + threading.Thread.__init__(self) + + def e(self, command, inputdata=None): + """execute command """ + self.command = command + self.inputdata = inputdata + self.result = "" + self.start() + self.join(10) + return self.result + + def run(self): + """Run a command and return standard output""" + pipe = subprocess.PIPE + popen = subprocess.Popen(self.command, stdout=pipe, stderr=pipe) + outputdata, errdata = popen.communicate(self.inputdata) + assert (popen.returncode == 0), \ + "Error running: %s\n\n%s" % (self.command, errdata) + self.result = outputdata + +class OCR(object): + def __init__(self): + self.logger = logging.getLogger("log") + + def load_image(self, image): + self.image = Image.open(image) + self.pixels = self.image.load() + self.result_captcha = '' + + def unload(self): + """delete all tmp images""" + pass + + def threshold(self, value): + self.image = self.image.point(lambda a: a * value + 10) + + def run(self, command, inputdata=None): + """Run a command and return standard output""" + # OLD METHOD + # pipe = subprocess.PIPE + # popen = subprocess.Popen(command, stdout=pipe, stderr=pipe) + # outputdata, errdata = popen.communicate(inputdata) + # assert (popen.returncode == 0), \ + # "Error running: %s\n\n%s" % (command, errdata) + # return outputdata + + thread = RunThread() + result = thread.e(command, inputdata) + return result + + def run_gocr(self): + tmp = tempfile.NamedTemporaryFile(suffix=".jpg") + self.image.save(tmp) + self.result_captcha = self.run(['gocr', tmp.name]).replace("\n", "") + + def run_tesser(self): + self.logger.debug("create tmp tif") + tmp = tempfile.NamedTemporaryFile(suffix=".tif") + self.logger.debug("create tmp txt") + tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") + self.logger.debug("save tiff") + self.image.save(tmp.name, 'TIFF') + self.logger.debug("run tesseract") + self.run(['tesseract', tmp.name, tmpTxt.name.replace(".txt", "")]) + self.logger.debug("read txt") + + with open(tmpTxt.name, 'r') as f: + self.result_captcha = f.read().replace("\n", "") + + def get_captcha(self): + raise NotImplementedError + + def to_greyscale(self): + if self.image.mode != 'L': + self.image = self.image.convert('L') + + self.pixels = self.image.load() + + def eval_black_white(self, limit): + self.pixels = self.image.load() + w, h = self.image.size + for x in xrange(w): + for y in xrange(h): + if self.pixels[x, y] > limit: + self.pixels[x, y] = 255 + else: + self.pixels[x, y] = 0 + + def clean(self, allowed): + pixels = self.pixels + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 255: continue + # no point in processing white pixels since we only want to remove black pixel + count = 0 + + try: + if pixels[x-1, y-1] != 255: count += 1 + if pixels[x-1, y] != 255: count += 1 + if pixels[x-1, y + 1] != 255: count += 1 + if pixels[x, y + 1] != 255: count += 1 + if pixels[x + 1, y + 1] != 255: count += 1 + if pixels[x + 1, y] != 255: count += 1 + if pixels[x + 1, y-1] != 255: count += 1 + if pixels[x, y-1] != 255: count += 1 + except: + pass + + # not enough neighbors are dark pixels so mark this pixel + # to be changed to white + if count < allowed: + pixels[x, y] = 1 + + # second pass: this time set all 1's to 255 (white) + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 1: pixels[x, y] = 255 + + self.pixels = pixels + + def derotate_by_average(self): + """rotate by checking each angle and guess most suitable""" + + w, h = self.image.size + pixels = self.pixels + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 155 + + highest = {} + counts = {} + + for angle in range(-45, 45): + + tmpimage = self.image.rotate(angle) + + pixels = tmpimage.load() + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + + count = {} + + for x in xrange(w): + count[x] = 0 + for y in xrange(h): + if pixels[x, y] == 155: + count[x] += 1 + + sum = 0 + cnt = 0 + + for x in count.values(): + if x != 0: + sum += x + cnt += 1 + + avg = sum / cnt + counts[angle] = cnt + highest[angle] = 0 + for x in count.values(): + if x > highest[angle]: + highest[angle] = x + + highest[angle] = highest[angle] - avg + + hkey = 0 + hvalue = 0 + + for key, value in highest.iteritems(): + if value > hvalue: + hkey = key + hvalue = value + + self.image = self.image.rotate(hkey) + pixels = self.image.load() + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + if pixels[x, y] == 155: + pixels[x, y] = 0 + + self.pixels = pixels + + def split_captcha_letters(self): + captcha = self.image + started = False + letters = [] + width, height = captcha.size + bottomY, topY = 0, height + pixels = captcha.load() + + for x in xrange(width): + black_pixel_in_col = False + for y in xrange(height): + if pixels[x, y] != 255: + if started == False: + started = True + firstX = x + lastX = x + + if y > bottomY: bottomY = y + if y < topY: topY = y + if x > lastX: lastX = x + + black_pixel_in_col = True + + if black_pixel_in_col == False and started == True: + rect = (firstX, topY, lastX, bottomY) + new_captcha = captcha.crop(rect) + + w, h = new_captcha.size + if w > 5 and h > 5: + letters.append(new_captcha) + + started = False + bottomY, topY = 0, height + + return letters + + def correct(self, values, var=None): + + if var: + result = var + else: + result = self.result_captcha + + for key, item in values.iteritems(): + + if key.__class__ == str: + result = result.replace(key, item) + else: + for expr in key: + result = result.replace(expr, item) + + if var: + return result + else: + self.result_captcha = result + + +if __name__ == '__main__': + ocr = OCR() + ocr.load_image("B.jpg") + ocr.to_greyscale() + ocr.eval_black_white(140) + ocr.derotate_by_avergage() + ocr.run_gocr() + print "GOCR", ocr.result_captcha + ocr.run_tesser() + print "Tesseract", ocr.result_captcha + ocr.image.save("derotated.jpg") + diff --git a/module/plugins/decrypter/SerienjunkiesOrg.py b/module/plugins/decrypter/SerienjunkiesOrg.py index 46f380857..a73779dd3 100644 --- a/module/plugins/decrypter/SerienjunkiesOrg.py +++ b/module/plugins/decrypter/SerienjunkiesOrg.py @@ -6,6 +6,25 @@ from time import sleep from module.Plugin import Plugin from module.BeautifulSoup import BeautifulSoup +from module.download_thread import CaptchaError + +from htmlentitydefs import name2codepoint as n2cp +def substitute_entity(match): + ent = match.group(2) + if match.group(1) == "#": + return unichr(int(ent)) + else: + cp = n2cp.get(ent) + if cp: + return unichr(cp) + else: + return match.group() + +def decode_htmlentities(string): + entity_re = re.compile("&(#?)(\d{1,5}|\w{1,8});") + return entity_re.subn(substitute_entity, string)[0] + + class SerienjunkiesOrg(Plugin): def __init__(self, parent): Plugin.__init__(self, parent) @@ -13,7 +32,7 @@ class SerienjunkiesOrg(Plugin): props['name'] = "SerienjunkiesOrg" props['type'] = "container" props['pattern'] = r"http://.*?serienjunkies.org/.*?" - props['version'] = "0.1" + props['version'] = "0.2" props['description'] = """serienjunkies.org Container Plugin""" props['author_name'] = ("mkaay") props['author_mail'] = ("mkaay@mkaay.de") @@ -21,6 +40,43 @@ class SerienjunkiesOrg(Plugin): self.parent = parent self.html = None self.multi_dl = False + + self.hosterMap = { + "rc": "RapidshareCom", + "ff": "FilefactoryCom", + "ut": "UploadedTo", + "ul": "UploadedTo", + "nl": "NetloadIn", + "rs": "RapidshareDe" + } + self.hosterMapReverse = dict((v,k) for k, v in self.hosterMap.iteritems()) + episodePattern = re.compile("^http://download.serienjunkies.org/f-.*?.html$") + oldStyleLink = re.compile("^http://serienjunkies.org/safe/(.*)$") + if re.match(episodePattern, self.parent.url) or re.match(oldStyleLink, self.parent.url): + self.decryptNow = False + else: + self.decryptNow = True + + def prepare(self, thread): + pyfile = self.parent + + self.want_reconnect = False + + pyfile.status.exists = self.file_exists() + + if not pyfile.status.exists: + raise Exception, "File not found" + return False + + pyfile.status.filename = self.get_file_name() + + pyfile.status.waituntil = self.time_plus_wait + pyfile.status.url = self.get_file_url() + pyfile.status.want_reconnect = self.want_reconnect + + thread.wait(self.parent) + + return True def getSJSrc(self, url): src = self.req.load(str(url)) @@ -31,9 +87,88 @@ class SerienjunkiesOrg(Plugin): def file_exists(self): return True + def waitForCaptcha(self, captchaData, imgType): + captchaManager = self.parent.core.captchaManager + task = captchaManager.newTask(self) + task.setCaptcha(captchaData, imgType) + task.setWaiting() + while not task.getStatus() == "done": + if not self.parent.core.isGUIConnected(): + task.removeTask() + raise CaptchaError + sleep(1) + result = task.getResult() + task.removeTask() + return result + + def handleSeason(self, url): + src = self.getSJSrc(url) + soup = BeautifulSoup(src) + post = soup.find("div", attrs={"class": "post-content"}) + ps = post.findAll("p") + hosterPattern = re.compile("^http://download\.serienjunkies\.org/f-.*?/([rcfultns]{2})_.*?\.html$") + preferredHoster = self.get_config("preferredHoster").split(",") + self.logger.debug("Preferred hoster: %s" % ", ".join(preferredHoster)) + groups = {} + gid = -1 + seasonName = soup.find("a", attrs={"rel":"bookmark"}).string + for p in ps: + if re.search("Dauer|Sprache|Format", str(p)): + var = p.findAll("strong") + opts = {"Dauer": "", "Uploader": "", "Sprache": "", "Format": "", u"Größe": ""} + for v in var: + n = decode_htmlentities(v.string) + val = v.nextSibling + val = val.encode("utf-8") + val = decode_htmlentities(val) + val = val.replace(" |", "") + n = n.strip() + n = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', n) + val = val.strip() + val = re.sub(r"^([:]?)(.*?)([:]?)$", r'\2', val) + opts[n.strip()] = val.strip() + gid += 1 + groups[gid] = {} + groups[gid]["ep"] = [] + groups[gid]["opts"] = opts + elif re.search("Download:", str(p)): + links1 = p.findAll("a", attrs={"href": hosterPattern}) + links2 = p.findAll("a", attrs={"href": re.compile("^http://serienjunkies.org/safe/.*$")}) + for link in links1 + links2: + groups[gid]["ep"].append(link["href"]) + packages = {} + for g in groups.values(): + links = [] + linklist = g["ep"] + package = "%s (%s, %s)" % (seasonName, g["opts"]["Format"], g["opts"]["Sprache"]) + linkgroups = {} + for link in linklist: + key = re.sub("^http://download\.serienjunkies\.org/f-.*?/([rcfultns]{2})_", "", link) + if not linkgroups.has_key(key): + linkgroups[key] = [] + linkgroups[key].append(link) + for group in linkgroups.values(): + print "group", group + for pHoster in preferredHoster: + print "phoster", pHoster + hmatch = False + for link in group: + print "link", link + m = hosterPattern.match(link) + if m: + if pHoster == self.hosterMap[m.group(1)]: + links.append(link) + hmatch = True + print "match" + break + if hmatch: + break + packages[package] = links + return packages + def handleEpisode(self, url): if not self.parent.core.isGUIConnected(): - return False + raise CaptchaError for i in range(3): src = self.getSJSrc(url) if not src.find("Du hast das Download-Limit überschritten! Bitte versuche es später nocheinmal.") == -1: @@ -45,17 +180,7 @@ class SerienjunkiesOrg(Plugin): captchaTag = soup.find(attrs={"src":re.compile("^/secure/")}) captchaUrl = "http://download.serienjunkies.org"+captchaTag["src"] captchaData = self.req.load(str(captchaUrl)) - captchaManager = self.parent.core.captchaManager - task = captchaManager.newTask(self) - task.setCaptcha(captchaData, "png") - task.setWaiting() - while not task.getStatus() == "done": - if not self.parent.core.isGUIConnected(): - task.removeTask() - return False - sleep(1) - result = task.getResult() - task.removeTask() + result = self.waitForCaptcha(captchaData, "png") url = "http://download.serienjunkies.org"+form["action"] sinp = form.find(attrs={"name":"s"}) @@ -73,6 +198,27 @@ class SerienjunkiesOrg(Plugin): links.append(self.handleFrame(frameUrl)) return links + def handleOldStyleLink(self, url): + if not self.parent.core.isGUIConnected(): + raise CaptchaError + for i in range(3): + sj = self.req.load(str(url)) + soup = BeautifulSoup(sj) + form = soup.find("form", attrs={"action":re.compile("^http://serienjunkies.org")}) + captchaTag = form.find(attrs={"src":re.compile("^/safe/secure/")}) + captchaUrl = "http://serienjunkies.org"+captchaTag["src"] + captchaData = self.req.load(str(captchaUrl)) + result = self.waitForCaptcha(captchaData, "png") + url = form["action"] + sinp = form.find(attrs={"name":"s"}) + + self.req.load(str(url), post={'s': sinp["value"], 'c': result, 'dl.start': "Download"}, cookies=False, just_header=True) + decrypted = self.req.lastEffectiveURL + if decrypted == str(url): + continue + return [decrypted] + return False + def handleFrame(self, url): self.req.load(str(url), cookies=False, just_header=True) return self.req.lastEffectiveURL @@ -80,9 +226,15 @@ class SerienjunkiesOrg(Plugin): def proceed(self, url, location): links = False episodePattern = re.compile("^http://download.serienjunkies.org/f-.*?.html$") + oldStyleLink = re.compile("^http://serienjunkies.org/safe/(.*)$") framePattern = re.compile("^http://download.serienjunkies.org/frame/go-.*?/$") + seasonPattern = re.compile("^http://serienjunkies.org/\?p=.*?$") if framePattern.match(url): links = [self.handleFrame(url)] elif episodePattern.match(url): links = self.handleEpisode(url) + elif oldStyleLink.match(url): + links = self.handleOldStyleLink(url) + elif seasonPattern.match(url): + links = self.handleSeason(url) self.links = links diff --git a/module/thread_list.py b/module/thread_list.py index ad0d0c8fb..d3eb4d203 100644 --- a/module/thread_list.py +++ b/module/thread_list.py @@ -110,15 +110,26 @@ class Thread_List(object): if pyfile.plugin.props['type'] == "container": newLinks = 0 if pyfile.plugin.links: - for link in pyfile.plugin.links: - newFile = self.list.collector.addLink(link) - self.list.packager.addFileToPackage(pyfile.package.data["id"], self.list.collector.popFile(newFile)) - newLinks += 1 - self.list.packager.pushPackage2Queue(pyfile.package.data["id"]) + if isinstance(pyfile.plugin.links, dict): + packmap = {} + for packname in pyfile.plugin.links.keys(): + packmap[packname] = self.list.packager.addNewPackage(packname) + for packname, links in pyfile.plugin.links.items(): + pid = packmap[packname] + for link in links: + newFile = self.list.collector.addLink(link) + self.list.packager.addFileToPackage(pid, self.list.collector.popFile(newFile)) + newLinks += 1 + else: + for link in pyfile.plugin.links: + newFile = self.list.collector.addLink(link) + self.list.packager.addFileToPackage(pyfile.package.data["id"], self.list.collector.popFile(newFile)) + newLinks += 1 + #self.list.packager.pushPackage2Queue(pyfile.package.data["id"]) self.list.packager.removeFileFromPackage(pyfile.id, pyfile.package.data["id"]) if newLinks: - self.parent.logger.info("Parsed link from %s: %i" % (pyfile.status.filename, newLinks)) + self.parent.logger.info("Parsed links from %s: %i" % (pyfile.status.filename, newLinks)) else: self.parent.logger.info("No links in %s" % pyfile.status.filename) #~ self.list.packager.removeFileFromPackage(pyfile.id, pyfile.package.id) -- cgit v1.2.3