diff options
Diffstat (limited to 'module/plugins/internal')
| -rw-r--r-- | module/plugins/internal/AbstractExtractor.py | 12 | ||||
| -rw-r--r-- | module/plugins/internal/MultiHoster.py | 189 | ||||
| -rw-r--r-- | module/plugins/internal/NetloadInOCR.py | 27 | ||||
| -rw-r--r-- | module/plugins/internal/OCR.py | 314 | ||||
| -rw-r--r-- | module/plugins/internal/ShareonlineBizOCR.py | 53 | ||||
| -rw-r--r-- | module/plugins/internal/SimpleHoster.py | 2 | ||||
| -rw-r--r-- | module/plugins/internal/UnRar.py | 15 |
7 files changed, 408 insertions, 204 deletions
diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index 2130f910e..3cd635eff 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -13,7 +13,7 @@ class WrongPassword(Exception): class AbtractExtractor: @staticmethod def checkDeps(): - """ Check if system statisfy dependencies + """ Check if system satisfies dependencies :return: boolean """ return True @@ -21,7 +21,7 @@ class AbtractExtractor: @staticmethod def getTargets(files_ids): """ Filter suited targets from list of filename id tuple list - :param files_ids: List of filepathes + :param files_ids: List of file paths :return: List of targets, id tuple list """ raise NotImplementedError @@ -30,10 +30,10 @@ class AbtractExtractor: def __init__(self, m, file, out, fullpath, overwrite, renice): """Initialize extractor for specific file - :param m: ExtractArchive Hook plugin - :param file: Absolute filepath + :param m: ExtractArchive addon plugin + :param file: Absolute file path :param out: Absolute path to destination directory - :param fullpath: extract to fullpath + :param fullpath: Extract to fullpath :param overwrite: Overwrite existing archives :param renice: Renice value """ @@ -52,7 +52,7 @@ class AbtractExtractor: def checkArchive(self): - """Check if password if needed. Raise ArchiveError if integrity is + """Check if password is needed. Raise ArchiveError if integrity is questionable. :return: boolean diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py deleted file mode 100644 index a8961aafc..000000000 --- a/module/plugins/internal/MultiHoster.py +++ /dev/null @@ -1,189 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.utils import remove_chars -from module.plugins.Hook import Hook - -class MultiHoster(Hook): - """ - Generic MultiHoster plugin - """ - - __version__ = "0.19" - - replacements = [("2shared.com", "twoshared.com"), ("4shared.com", "fourshared.com"), ("cloudnator.com", "shragle.com"), - ("ifile.it", "filecloud.io"), ("easy-share.com","crocko.com"), ("freakshare.net","freakshare.com"), - ("hellshare.com", "hellshare.cz"), ("share-rapid.cz","sharerapid.com"), ("sharerapid.cz","sharerapid.com"), - ("ul.to","uploaded.to"), ("uploaded.net","uploaded.to"), ("1fichier.com", "onefichier.com")] - ignored = [] - interval = 24 * 60 * 60 # reload hosters daily - - def setup(self): - self.hosters = [] - self.supported = [] - self.new_supported = [] - - def getConfig(self, option, default = ''): - """getConfig with default value - sublass may not implements all config options""" - try: - return self.getConf(option) - except KeyError: - return default - - def getHosterCached(self): - if not self.hosters: - - try: - hosterSet = self.toHosterSet(self.getHoster()) - set(self.ignored) - except Exception, e: - self.logError("%s" % str(e)) - return [] - - try: - configMode = self.getConfig('hosterListMode', 'all') - if configMode in ("listed", "unlisted"): - configSet = self.toHosterSet(self.getConfig('hosterList', '').replace('|',',').replace(';',',').split(',')) - - if configMode == "listed": - hosterSet &= configSet - else: - hosterSet -= configSet - - except Exception, e: - self.logError("%s" % str(e)) - - self.hosters = list(hosterSet) - - return self.hosters - - def toHosterSet(self, hosters): - hosters = set((str(x).strip().lower() for x in hosters)) - - for rep in self.replacements: - if rep[0] in hosters: - hosters.remove(rep[0]) - hosters.add(rep[1]) - - hosters.discard('') - return hosters - - def getHoster(self): - """Load list of supported hoster - - :return: List of domain names - """ - raise NotImplementedError - - def coreReady(self): - if self.cb: - self.core.scheduler.removeJob(self.cb) - - self.setConfig("activated", True) # config not in sync after plugin reload - - cfg_interval = self.getConfig("interval", None) # reload interval in hours - if cfg_interval is not None: - self.interval = cfg_interval * 60 * 60 - - if self.interval: - self._periodical() - else: - self.periodical() - - def initPeriodical(self): - pass - - def periodical(self): - """reload hoster list periodically""" - self.logInfo("Reloading supported hoster list") - - old_supported = self.supported - self.supported, self.new_supported, self.hosters = [], [], [] - - self.overridePlugins() - - old_supported = [hoster for hoster in old_supported if hoster not in self.supported] - if old_supported: - self.logDebug("UNLOAD: %s" % ", ".join(old_supported)) - for hoster in old_supported: - self.unloadHoster(hoster) - - def overridePlugins(self): - pluginMap = {} - for name in self.core.pluginManager.hosterPlugins.keys(): - pluginMap[name.lower()] = name - - accountList = [ name.lower() for name, data in self.core.accountManager.accounts.items() if data ] - excludedList = [] - - for hoster in self.getHosterCached(): - name = remove_chars(hoster.lower(), "-.") - - if name in accountList: - excludedList.append(hoster) - else: - if name in pluginMap: - self.supported.append(pluginMap[name]) - else: - self.new_supported.append(hoster) - - if not self.supported and not self.new_supported: - self.logError(_("No Hoster loaded")) - return - - module = self.core.pluginManager.getPlugin(self.__name__) - klass = getattr(module, self.__name__) - - # inject plugin plugin - self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) - for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - dict["new_module"] = module - dict["new_name"] = self.__name__ - - if excludedList: - self.logInfo("The following hosters were not overwritten - account exists: %s" % ", ".join(sorted(excludedList))) - - if self.new_supported: - self.logDebug("New Hosters: %s" % ", ".join(sorted(self.new_supported))) - - # create new regexp - regexp = r".*(%s).*" % "|".join([x.replace(".", "\\.") for x in self.new_supported]) - if hasattr(klass, "__pattern__") and isinstance(klass.__pattern__, basestring) and '://' in klass.__pattern__: - regexp = r"%s|%s" % (klass.__pattern__, regexp) - - self.logDebug("Regexp: %s" % regexp) - - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict["pattern"] = regexp - dict["re"] = re.compile(regexp) - - def unloadHoster(self, hoster): - dict = self.core.pluginManager.hosterPlugins[hoster] - if "module" in dict: - del dict["module"] - - if "new_module" in dict: - del dict["new_module"] - del dict["new_name"] - - def unload(self): - """Remove override for all hosters. Scheduler job is removed by hookmanager""" - for hoster in self.supported: - self.unloadHoster(hoster) - - # reset pattern - klass = getattr(self.core.pluginManager.getPlugin(self.__name__), self.__name__) - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict["pattern"] = getattr(klass, '__pattern__', r"^unmatchable$") - dict["re"] = re.compile(dict["pattern"]) - - def downloadFailed(self, pyfile): - """remove plugin override if download fails but not if file is offline/temp.offline""" - if pyfile.hasStatus("failed") and self.getConfig("unloadFailing", True): - hdict = self.core.pluginManager.hosterPlugins[pyfile.pluginname] - if "new_name" in hdict and hdict['new_name'] == self.__name__: - self.logDebug("Unload MultiHoster", pyfile.pluginname, hdict) - self.unloadHoster(pyfile.pluginname) - pyfile.setStatus("queued")
\ No newline at end of file diff --git a/module/plugins/internal/NetloadInOCR.py b/module/plugins/internal/NetloadInOCR.py new file mode 100644 index 000000000..e50978701 --- /dev/null +++ b/module/plugins/internal/NetloadInOCR.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from OCR import OCR + +class NetloadInOCR(OCR): + __version__ = 0.1 + + def __init__(self): + OCR.__init__(self) + + def get_captcha(self, image): + self.load_image(image) + self.to_greyscale() + self.clean(3) + self.clean(3) + self.run_tesser(True, True, False, False) + + self.result_captcha = self.result_captcha.replace(" ", "")[:4] # cut to 4 numbers + + return self.result_captcha + +if __name__ == '__main__': + import urllib + ocr = NetloadInOCR() + urllib.urlretrieve("http://netload.in/share/includes/captcha.php", "captcha.png") + + print ocr.get_captcha('captcha.png') diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py new file mode 100644 index 000000000..9f8b7ef8c --- /dev/null +++ b/module/plugins/internal/OCR.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2009 kingzero, RaNaN +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see <http://www.gnu.org/licenses/>. +# +### +from __future__ import with_statement +import os +from os.path import join +from os.path import abspath +import logging +import subprocess +#import tempfile + +import Image +import TiffImagePlugin +import PngImagePlugin +import GifImagePlugin +import JpegImagePlugin + + +class OCR(object): + __version__ = 0.1 + + def __init__(self): + self.logger = logging.getLogger("log") + + def load_image(self, image): + self.image = Image.open(image) + self.pixels = self.image.load() + self.result_captcha = '' + + def unload(self): + """delete all tmp images""" + pass + + def threshold(self, value): + self.image = self.image.point(lambda a: a * value + 10) + + def run(self, command): + """Run a command""" + + popen = subprocess.Popen(command, bufsize = -1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + popen.wait() + output = popen.stdout.read() +" | "+ popen.stderr.read() + popen.stdout.close() + popen.stderr.close() + self.logger.debug("Tesseract ReturnCode %s Output: %s" % (popen.returncode, output)) + + def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True): + #self.logger.debug("create tmp tif") + + + #tmp = tempfile.NamedTemporaryFile(suffix=".tif") + tmp = open(join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") + tmp.close() + #self.logger.debug("create tmp txt") + #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") + tmpTxt = open(join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") + tmpTxt.close() + + self.logger.debug("save tiff") + self.image.save(tmp.name, 'TIFF') + + if os.name == "nt": + tessparams = [join(pypath,"tesseract","tesseract.exe")] + else: + tessparams = ['tesseract'] + + tessparams.extend( [abspath(tmp.name), abspath(tmpTxt.name).replace(".txt", "")] ) + + if subset and (digits or lowercase or uppercase): + #self.logger.debug("create temp subset config") + #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") + tmpSub = open(join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") + tmpSub.write("tessedit_char_whitelist ") + if digits: + tmpSub.write("0123456789") + if lowercase: + tmpSub.write("abcdefghijklmnopqrstuvwxyz") + if uppercase: + tmpSub.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + tmpSub.write("\n") + tessparams.append("nobatch") + tessparams.append(abspath(tmpSub.name)) + tmpSub.close() + + self.logger.debug("run tesseract") + self.run(tessparams) + self.logger.debug("read txt") + + try: + with open(tmpTxt.name, 'r') as f: + self.result_captcha = f.read().replace("\n", "") + except: + self.result_captcha = "" + + self.logger.debug(self.result_captcha) + try: + os.remove(tmp.name) + os.remove(tmpTxt.name) + if subset and (digits or lowercase or uppercase): + os.remove(tmpSub.name) + except: + pass + + def get_captcha(self, name): + raise NotImplementedError + + def to_greyscale(self): + if self.image.mode != 'L': + self.image = self.image.convert('L') + + self.pixels = self.image.load() + + def eval_black_white(self, limit): + self.pixels = self.image.load() + w, h = self.image.size + for x in xrange(w): + for y in xrange(h): + if self.pixels[x, y] > limit: + self.pixels[x, y] = 255 + else: + self.pixels[x, y] = 0 + + def clean(self, allowed): + pixels = self.pixels + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 255: continue + # no point in processing white pixels since we only want to remove black pixel + count = 0 + + try: + if pixels[x-1, y-1] != 255: count += 1 + if pixels[x-1, y] != 255: count += 1 + if pixels[x-1, y + 1] != 255: count += 1 + if pixels[x, y + 1] != 255: count += 1 + if pixels[x + 1, y + 1] != 255: count += 1 + if pixels[x + 1, y] != 255: count += 1 + if pixels[x + 1, y-1] != 255: count += 1 + if pixels[x, y-1] != 255: count += 1 + except: + pass + + # not enough neighbors are dark pixels so mark this pixel + # to be changed to white + if count < allowed: + pixels[x, y] = 1 + + # second pass: this time set all 1's to 255 (white) + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 1: pixels[x, y] = 255 + + self.pixels = pixels + + def derotate_by_average(self): + """rotate by checking each angle and guess most suitable""" + + w, h = self.image.size + pixels = self.pixels + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 155 + + highest = {} + counts = {} + + for angle in range(-45, 45): + + tmpimage = self.image.rotate(angle) + + pixels = tmpimage.load() + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + + count = {} + + for x in xrange(w): + count[x] = 0 + for y in xrange(h): + if pixels[x, y] == 155: + count[x] += 1 + + sum = 0 + cnt = 0 + + for x in count.values(): + if x != 0: + sum += x + cnt += 1 + + avg = sum / cnt + counts[angle] = cnt + highest[angle] = 0 + for x in count.values(): + if x > highest[angle]: + highest[angle] = x + + highest[angle] = highest[angle] - avg + + hkey = 0 + hvalue = 0 + + for key, value in highest.iteritems(): + if value > hvalue: + hkey = key + hvalue = value + + self.image = self.image.rotate(hkey) + pixels = self.image.load() + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + if pixels[x, y] == 155: + pixels[x, y] = 0 + + self.pixels = pixels + + def split_captcha_letters(self): + captcha = self.image + started = False + letters = [] + width, height = captcha.size + bottomY, topY = 0, height + pixels = captcha.load() + + for x in xrange(width): + black_pixel_in_col = False + for y in xrange(height): + if pixels[x, y] != 255: + if not started: + started = True + firstX = x + lastX = x + + if y > bottomY: bottomY = y + if y < topY: topY = y + if x > lastX: lastX = x + + black_pixel_in_col = True + + if black_pixel_in_col == False and started == True: + rect = (firstX, topY, lastX, bottomY) + new_captcha = captcha.crop(rect) + + w, h = new_captcha.size + if w > 5 and h > 5: + letters.append(new_captcha) + + started = False + bottomY, topY = 0, height + + return letters + + def correct(self, values, var=None): + + if var: + result = var + else: + result = self.result_captcha + + for key, item in values.iteritems(): + + if key.__class__ == str: + result = result.replace(key, item) + else: + for expr in key: + result = result.replace(expr, item) + + if var: + return result + else: + self.result_captcha = result + + +if __name__ == '__main__': + ocr = OCR() + ocr.load_image("B.jpg") + ocr.to_greyscale() + ocr.eval_black_white(140) + ocr.derotate_by_average() + ocr.run_tesser() + print "Tesseract", ocr.result_captcha + ocr.image.save("derotated.jpg") + diff --git a/module/plugins/internal/ShareonlineBizOCR.py b/module/plugins/internal/ShareonlineBizOCR.py new file mode 100644 index 000000000..c5c2e92e8 --- /dev/null +++ b/module/plugins/internal/ShareonlineBizOCR.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2009 kingzero, RaNaN +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see <http://www.gnu.org/licenses/>. +# +### +from OCR import OCR + +class ShareonlineBizOCR(OCR): + __version__ = 0.1 + + def __init__(self): + OCR.__init__(self) + + def get_captcha(self, image): + self.load_image(image) + self.to_greyscale() + self.image = self.image.resize((160, 50)) + self.pixels = self.image.load() + self.threshold(1.85) + #self.eval_black_white(240) + #self.derotate_by_average() + + letters = self.split_captcha_letters() + + final = "" + for letter in letters: + self.image = letter + self.run_tesser(True, True, False, False) + final += self.result_captcha + + return final + + #tesseract at 60% + +if __name__ == '__main__': + import urllib + ocr = ShareonlineBizOCR() + urllib.urlretrieve("http://www.share-online.biz/captcha.php", "captcha.jpeg") + print ocr.get_captcha('captcha.jpeg') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 69ed57ff8..7b1d7323a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -158,7 +158,7 @@ class SimpleHoster(Hoster): or FILE_NAME_PATTERN = r'(?P<N>file_name)' and FILE_SIZE_PATTERN = r'(?P<S>file_size) (?P<U>units)' FILE_OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN = r'Server maintainance' + TEMP_OFFLINE_PATTERN = r'Server maintenance' """ FILE_SIZE_REPLACEMENTS = [] diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index da8e7cf3d..7becd663c 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -19,12 +19,11 @@ import os import re -from os.path import join from glob import glob from subprocess import Popen, PIPE from string import digits -from module.utils import save_join, decode +from module.utils.fs import save_join, decode, fs_encode from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError class UnRar(AbtractExtractor): @@ -41,7 +40,7 @@ class UnRar(AbtractExtractor): @staticmethod def checkDeps(): if os.name == "nt": - UnRar.CMD = join(pypath, "UnRAR.exe") + UnRar.CMD = save_join(pypath, "UnRAR.exe") p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) p.communicate() else: @@ -82,7 +81,7 @@ class UnRar(AbtractExtractor): self.password = "" #save the correct password def checkArchive(self): - p = self.call_unrar("l", "-v", self.file) + p = self.call_unrar("l", "-v", fs_encode(self.file)) out, err = p.communicate() if self.re_wrongpwd.search(err): self.passwordProtected = True @@ -104,7 +103,7 @@ class UnRar(AbtractExtractor): def checkPassword(self, password): #at this point we can only verify header protected files if self.headerProtected: - p = self.call_unrar("l", "-v", self.file, password=password) + p = self.call_unrar("l", "-v", fs_encode(self.file), password=password) out, err = p.communicate() if self.re_wrongpwd.search(err): return False @@ -115,7 +114,7 @@ class UnRar(AbtractExtractor): def extract(self, progress, password=None): command = "x" if self.fullpath else "e" - p = self.call_unrar(command, self.file, self.out, password=password) + p = self.call_unrar(command, fs_encode(self.file), self.out, password=password) renice(p.pid, self.renice) progress(0) @@ -163,7 +162,7 @@ class UnRar(AbtractExtractor): def listContent(self): command = "vb" if self.fullpath else "lb" - p = self.call_unrar(command, "-v", self.file, password=self.password) + p = self.call_unrar(command, "-v", fs_encode(self.file), password=self.password) out, err = p.communicate() if "Cannot open" in err: @@ -198,7 +197,7 @@ class UnRar(AbtractExtractor): #NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) - self.m.logDebug(" ".join(call)) + self.m.logDebug(" ".join([decode(arg) for arg in call])) p = Popen(call, stdout=PIPE, stderr=PIPE) |