diff options
Diffstat (limited to 'pyload/plugins/base')
-rw-r--r-- | pyload/plugins/base/Account.py | 281 | ||||
-rw-r--r-- | pyload/plugins/base/Container.py | 61 | ||||
-rw-r--r-- | pyload/plugins/base/Crypter.py | 61 | ||||
-rw-r--r-- | pyload/plugins/base/Hook.py | 149 | ||||
-rw-r--r-- | pyload/plugins/base/Hoster.py | 20 | ||||
-rw-r--r-- | pyload/plugins/base/OCR.py | 299 | ||||
-rw-r--r-- | pyload/plugins/base/__init__.py | 0 |
7 files changed, 871 insertions, 0 deletions
diff --git a/pyload/plugins/base/Account.py b/pyload/plugins/base/Account.py new file mode 100644 index 000000000..e338f6b26 --- /dev/null +++ b/pyload/plugins/base/Account.py @@ -0,0 +1,281 @@ +# -*- coding: utf-8 -*- + +from random import choice +from time import time +from traceback import print_exc +from threading import RLock + +from pyload.plugins.Plugin import Base +from pyload.utils import compare_time, parseFileSize, lock + + +class WrongPassword(Exception): + pass + + +class Account(Base): + """ + Base class for every Account plugin. + Just overwrite `login` and cookies will be stored and account becomes accessible in\ + associated hoster plugin. Plugin should also provide `loadAccountInfo` + """ + __name__ = "Account" + __type__ = "account" + __version__ = "0.3" + + __description__ = """Base account plugin""" + __author_name__ = "mkaay" + __author_mail__ = "mkaay@mkaay.de" + + #: after that time (in minutes) pyload will relogin the account + login_timeout = 10 * 60 + #: after that time (in minutes) account data will be reloaded + info_threshold = 10 * 60 + + + def __init__(self, manager, accounts): + Base.__init__(self, manager.core) + + self.manager = manager + self.accounts = {} + self.infos = {} # cache for account information + self.lock = RLock() + + self.timestamps = {} + self.setAccounts(accounts) + self.init() + + def init(self): + pass + + def login(self, user, data, req): + """login into account, the cookies will be saved so user can be recognized + + :param user: loginname + :param data: data dictionary + :param req: `Request` instance + """ + pass + + @lock + def _login(self, user, data): + # set timestamp for login + self.timestamps[user] = time() + + req = self.getAccountRequest(user) + try: + self.login(user, data, req) + except WrongPassword: + self.logWarning( + _("Could not login with account %(user)s | %(msg)s") % {"user": user + , "msg": _("Wrong Password")}) + success = data['valid'] = False + except Exception, e: + self.logWarning( + _("Could not login with account %(user)s | %(msg)s") % {"user": user + , "msg": e}) + success = data['valid'] = False + if self.core.debug: + print_exc() + else: + success = True + finally: + if req: + req.close() + return success + + def relogin(self, user): + req = self.getAccountRequest(user) + if req: + req.cj.clear() + req.close() + if user in self.infos: + del self.infos[user] #delete old information + + return self._login(user, self.accounts[user]) + + def setAccounts(self, accounts): + self.accounts = accounts + for user, data in self.accounts.iteritems(): + self._login(user, data) + self.infos[user] = {} + + def updateAccounts(self, user, password=None, options={}): + """ updates account and return true if anything changed """ + + if user in self.accounts: + self.accounts[user]['valid'] = True #do not remove or accounts will not login + if password: + self.accounts[user]['password'] = password + self.relogin(user) + return True + if options: + before = self.accounts[user]['options'] + self.accounts[user]['options'].update(options) + return self.accounts[user]['options'] != before + else: + self.accounts[user] = {"password": password, "options": options, "valid": True} + self._login(user, self.accounts[user]) + return True + + def removeAccount(self, user): + if user in self.accounts: + del self.accounts[user] + if user in self.infos: + del self.infos[user] + if user in self.timestamps: + del self.timestamps[user] + + @lock + def getAccountInfo(self, name, force=False): + """retrieve account infos for an user, do **not** overwrite this method!\\ + just use it to retrieve infos in hoster plugins. see `loadAccountInfo` + + :param name: username + :param force: reloads cached account information + :return: dictionary with information + """ + data = Account.loadAccountInfo(self, name) + + if force or name not in self.infos: + self.logDebug("Get Account Info for %s" % name) + req = self.getAccountRequest(name) + + try: + infos = self.loadAccountInfo(name, req) + if not type(infos) == dict: + raise Exception("Wrong return format") + except Exception, e: + infos = {"error": str(e)} + + if req: req.close() + + self.logDebug("Account Info: %s" % infos) + + infos['timestamp'] = time() + self.infos[name] = infos + elif "timestamp" in self.infos[name] and self.infos[name][ + "timestamp"] + self.info_threshold * 60 < time(): + self.logDebug("Reached timeout for account data") + self.scheduleRefresh(name) + + data.update(self.infos[name]) + return data + + def isPremium(self, user): + info = self.getAccountInfo(user) + return info['premium'] + + def loadAccountInfo(self, name, req=None): + """this should be overwritten in account plugin,\ + and retrieving account information for user + + :param name: + :param req: `Request` instance + :return: + """ + return { + "validuntil": None, # -1 for unlimited + "login": name, + #"password": self.accounts[name]['password'], #@XXX: security + "options": self.accounts[name]['options'], + "valid": self.accounts[name]['valid'], + "trafficleft": None, # in kb, -1 for unlimited + "maxtraffic": None, + "premium": True, #useful for free accounts + "timestamp": 0, #time this info was retrieved + "type": self.__name__, + } + + def getAllAccounts(self, force=False): + return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()] + + def getAccountRequest(self, user=None): + if not user: + user, data = self.selectAccount() + if not user: + return None + + req = self.core.requestFactory.getRequest(self.__name__, user) + return req + + def getAccountCookies(self, user=None): + if not user: + user, data = self.selectAccount() + if not user: + return None + + cj = self.core.requestFactory.getCookieJar(self.__name__, user) + return cj + + def getAccountData(self, user): + return self.accounts[user] + + def selectAccount(self): + """ returns an valid account name and data""" + usable = [] + for user, data in self.accounts.iteritems(): + if not data['valid']: continue + + if "time" in data['options'] and data['options']['time']: + time_data = "" + try: + time_data = data['options']['time'][0] + start, end = time_data.split("-") + if not compare_time(start.split(":"), end.split(":")): + continue + except: + self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + + if user in self.infos: + if "validuntil" in self.infos[user]: + if self.infos[user]['validuntil'] > 0 and time() > self.infos[user]['validuntil']: + continue + if "trafficleft" in self.infos[user]: + if self.infos[user]['trafficleft'] == 0: + continue + + usable.append((user, data)) + + if not usable: return None, None + return choice(usable) + + def canUse(self): + return False if self.selectAccount() == (None, None) else True + + def parseTraffic(self, string): #returns kbyte + return parseFileSize(string) / 1024 + + def wrongPassword(self): + raise WrongPassword + + def empty(self, user): + if user in self.infos: + self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user) + + self.infos[user].update({"trafficleft": 0}) + self.scheduleRefresh(user, 30 * 60) + + def expired(self, user): + if user in self.infos: + self.logWarning(_("Account %s is expired, checking again in 1h") % user) + + self.infos[user].update({"validuntil": time() - 1}) + self.scheduleRefresh(user, 60 * 60) + + def scheduleRefresh(self, user, time=0, force=True): + """ add task to refresh account info to sheduler """ + self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) + self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) + + @lock + def checkLogin(self, user): + """ checks if user is still logged in """ + if user in self.timestamps: + if self.login_timeout > 0 and self.timestamps[user] + self.login_timeout * 60 < time(): + self.logDebug("Reached login timeout for %s" % user) + return self.relogin(user) + else: + return True + else: + return False diff --git a/pyload/plugins/base/Container.py b/pyload/plugins/base/Container.py new file mode 100644 index 000000000..2059ae9b2 --- /dev/null +++ b/pyload/plugins/base/Container.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +import re + +from os import remove +from os.path import basename, exists + +from pyload.plugins.base.Crypter import Crypter +from pyload.utils import safe_join + + +class Container(Crypter): + __name__ = "Container" + __type__ = "container" + __version__ = "0.1" + + __pattern__ = None + + __description__ = """Base container decrypter plugin""" + __author_name__ = "mkaay" + __author_mail__ = "mkaay@mkaay.de" + + + def preprocessing(self, thread): + """prepare""" + + self.setup() + self.thread = thread + + self.loadToDisk() + + self.decrypt(self.pyfile) + self.deleteTmp() + + self.createPackages() + + + def loadToDisk(self): + """loads container to disk if its stored remotely and overwrite url, + or check existent on several places at disk""" + + if self.pyfile.url.startswith("http"): + self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] + content = self.load(self.pyfile.url) + self.pyfile.url = safe_join(self.config['general']['download_folder'], self.pyfile.name) + f = open(self.pyfile.url, "wb" ) + f.write(content) + f.close() + + else: + self.pyfile.name = basename(self.pyfile.url) + if not exists(self.pyfile.url): + if exists(safe_join(pypath, self.pyfile.url)): + self.pyfile.url = safe_join(pypath, self.pyfile.url) + else: + self.fail(_("File not exists.")) + + + def deleteTmp(self): + if self.pyfile.name.startswith("tmp_"): + remove(self.pyfile.url) diff --git a/pyload/plugins/base/Crypter.py b/pyload/plugins/base/Crypter.py new file mode 100644 index 000000000..7bb48d607 --- /dev/null +++ b/pyload/plugins/base/Crypter.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +from pyload.plugins.Plugin import Plugin + + +class Crypter(Plugin): + __name__ = "Crypter" + __type__ = "crypter" + __version__ = "0.1" + + __pattern__ = None + + __description__ = """Base decrypter plugin""" + __author_name__ = "mkaay" + __author_mail__ = "mkaay@mkaay.de" + + + def __init__(self, pyfile): + Plugin.__init__(self, pyfile) + + #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) + self.packages = [] + + #: List of urls, pyLoad will generate packagenames + self.urls = [] + + self.multiDL = True + self.limitDL = 0 + + + def process(self, pyfile): + """ main method """ + self.decrypt(pyfile) + self.createPackages() + + + def decrypt(self, pyfile): + raise NotImplementedError + + + def createPackages(self): + """ create new packages from self.packages """ + for pack in self.packages: + + name, links, folder = pack + + self.logDebug("Parsed package %(name)s with %(len)d links" % {"name": name, "len": len(links)}) + + links = [x.decode("utf-8") for x in links] + + pid = self.api.addPackage(name, links, self.pyfile.package().queue) + + if name != folder is not None: + self.api.setPackageData(pid, {"folder": folder}) #: Due to not break API addPackage method right now + self.logDebug("Set package %(name)s folder to %(folder)s" % {"name": name, "folder": folder}) + + if self.pyfile.package().password: + self.api.setPackageData(pid, {"password": self.pyfile.package().password}) + + if self.urls: + self.api.generateAndAddPackages(self.urls) diff --git a/pyload/plugins/base/Hook.py b/pyload/plugins/base/Hook.py new file mode 100644 index 000000000..b9ffbc647 --- /dev/null +++ b/pyload/plugins/base/Hook.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- + +from traceback import print_exc + +from pyload.plugins.Plugin import Base + + +class Expose(object): + """ used for decoration to declare rpc services """ + + def __new__(cls, f, *args, **kwargs): + hookManager.addRPC(f.__module__, f.func_name, f.func_doc) + return f + + +def threaded(f): + + def run(*args,**kwargs): + hookManager.startThread(f, *args, **kwargs) + return run + + +class Hook(Base): + """ + Base class for hook plugins. + """ + __name__ = "Hook" + __type__ = "hook" + __version__ = "0.2" + + __config__ = [("name", "type", "desc", "default")] + + __description__ = """Interface for hook""" + __author_name__ = ("mkaay", "RaNaN") + __author_mail__ = ("mkaay@mkaay.de", "RaNaN@pyload.org") + + #: automatically register event listeners for functions, attribute will be deleted dont use it yourself + event_map = None + + # Alternative to event_map + #: List of events the plugin can handle, name the functions exactly like eventname. + event_list = None # dont make duplicate entries in event_map + + #: periodic call interval in secondc + interval = 60 + + + def __init__(self, core, manager): + Base.__init__(self, core) + + #: Provide information in dict here, usable by API `getInfo` + self.info = None + + #: Callback of periodical job task, used by hookmanager + self.cb = None + + #: `HookManager` + self.manager = manager + + #register events + if self.event_map: + for event, funcs in self.event_map.iteritems(): + if type(funcs) in (list, tuple): + for f in funcs: + self.manager.addEvent(event, getattr(self,f)) + else: + self.manager.addEvent(event, getattr(self,funcs)) + + #delete for various reasons + self.event_map = None + + if self.event_list: + for f in self.event_list: + self.manager.addEvent(f, getattr(self,f)) + + self.event_list = None + + self.setup() + self.initPeriodical() + + + def initPeriodical(self): + if self.interval >=1: + self.cb = self.core.scheduler.addJob(0, self._periodical, threaded=False) + + def _periodical(self): + try: + if self.isActivated(): self.periodical() + except Exception, e: + self.logError(_("Error executing hooks: %s") % str(e)) + if self.core.debug: + print_exc() + + self.cb = self.core.scheduler.addJob(self.interval, self._periodical, threaded=False) + + + def __repr__(self): + return "<Hook %s>" % self.__name__ + + def setup(self): + """ more init stuff if needed """ + pass + + def unload(self): + """ called when hook was deactivated """ + pass + + def isActivated(self): + """ checks if hook is activated""" + return self.config.getPlugin(self.__name__, "activated") + + + #event methods - overwrite these if needed + def coreReady(self): + pass + + def coreExiting(self): + pass + + def downloadPreparing(self, pyfile): + pass + + def downloadFinished(self, pyfile): + pass + + def downloadFailed(self, pyfile): + pass + + def packageFinished(self, pypack): + pass + + def beforeReconnecting(self, ip): + pass + + def afterReconnecting(self, ip): + pass + + def periodical(self): + pass + + def newCaptchaTask(self, task): + """ new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """ + pass + + def captchaCorrect(self, task): + pass + + def captchaInvalid(self, task): + pass diff --git a/pyload/plugins/base/Hoster.py b/pyload/plugins/base/Hoster.py new file mode 100644 index 000000000..23369deec --- /dev/null +++ b/pyload/plugins/base/Hoster.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +from pyload.plugins.Plugin import Plugin + + +def getInfo(self): + #result = [ .. (name, size, status, url) .. ] + return + + +class Hoster(Plugin): + __name__ = "Hoster" + __type__ = "hoster" + __version__ = "0.1" + + __pattern__ = None + + __description__ = """Base hoster plugin""" + __author_name__ = "mkaay" + __author_mail__ = "mkaay@mkaay.de" diff --git a/pyload/plugins/base/OCR.py b/pyload/plugins/base/OCR.py new file mode 100644 index 000000000..0991184f3 --- /dev/null +++ b/pyload/plugins/base/OCR.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement +import os +import logging +import subprocess + +from os.path import abspath, join +from PIL import Image +from PIL import TiffImagePlugin +from PIL import PngImagePlugin +from PIL import GifImagePlugin +from PIL import JpegImagePlugin + + +class OCR(object): + __name__ = "OCR" + __type__ = "ocr" + __version__ = "0.1" + + __description__ = """OCR base plugin""" + __author_name__ = "pyLoad Team" + __author_mail__ = "admin@pyload.org" + + + def __init__(self): + self.logger = logging.getLogger("log") + + def load_image(self, image): + self.image = Image.open(image) + self.pixels = self.image.load() + self.result_captcha = '' + + def unload(self): + """delete all tmp images""" + pass + + def threshold(self, value): + self.image = self.image.point(lambda a: a * value + 10) + + def run(self, command): + """Run a command""" + + popen = subprocess.Popen(command, bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + popen.wait() + output = popen.stdout.read() + " | " + popen.stderr.read() + popen.stdout.close() + popen.stderr.close() + self.logger.debug("Tesseract ReturnCode %s Output: %s" % (popen.returncode, output)) + + def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True): + #self.logger.debug("create tmp tif") + #tmp = tempfile.NamedTemporaryFile(suffix=".tif") + tmp = open(join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") + tmp.close() + #self.logger.debug("create tmp txt") + #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") + tmpTxt = open(join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") + tmpTxt.close() + + self.logger.debug("save tiff") + self.image.save(tmp.name, 'TIFF') + + if os.name == "nt": + tessparams = [join(pypath, "tesseract", "tesseract.exe")] + else: + tessparams = ['tesseract'] + + tessparams.extend([abspath(tmp.name), abspath(tmpTxt.name).replace(".txt", "")]) + + if subset and (digits or lowercase or uppercase): + #self.logger.debug("create temp subset config") + #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") + tmpSub = open(join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") + tmpSub.write("tessedit_char_whitelist ") + if digits: + tmpSub.write("0123456789") + if lowercase: + tmpSub.write("abcdefghijklmnopqrstuvwxyz") + if uppercase: + tmpSub.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + tmpSub.write("\n") + tessparams.append("nobatch") + tessparams.append(abspath(tmpSub.name)) + tmpSub.close() + + self.logger.debug("run tesseract") + self.run(tessparams) + self.logger.debug("read txt") + + try: + with open(tmpTxt.name, 'r') as f: + self.result_captcha = f.read().replace("\n", "") + except: + self.result_captcha = "" + + self.logger.debug(self.result_captcha) + try: + os.remove(tmp.name) + os.remove(tmpTxt.name) + if subset and (digits or lowercase or uppercase): + os.remove(tmpSub.name) + except: + pass + + def get_captcha(self, name): + raise NotImplementedError + + def to_greyscale(self): + if self.image.mode != 'L': + self.image = self.image.convert('L') + + self.pixels = self.image.load() + + def eval_black_white(self, limit): + self.pixels = self.image.load() + w, h = self.image.size + for x in xrange(w): + for y in xrange(h): + if self.pixels[x, y] > limit: + self.pixels[x, y] = 255 + else: + self.pixels[x, y] = 0 + + def clean(self, allowed): + pixels = self.pixels + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 255: + continue + # No point in processing white pixels since we only want to remove black pixel + count = 0 + + try: + if pixels[x - 1, y - 1] != 255: + count += 1 + if pixels[x - 1, y] != 255: + count += 1 + if pixels[x - 1, y + 1] != 255: + count += 1 + if pixels[x, y + 1] != 255: + count += 1 + if pixels[x + 1, y + 1] != 255: + count += 1 + if pixels[x + 1, y] != 255: + count += 1 + if pixels[x + 1, y - 1] != 255: + count += 1 + if pixels[x, y - 1] != 255: + count += 1 + except: + pass + + # not enough neighbors are dark pixels so mark this pixel + # to be changed to white + if count < allowed: + pixels[x, y] = 1 + + # second pass: this time set all 1's to 255 (white) + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 1: + pixels[x, y] = 255 + + self.pixels = pixels + + def derotate_by_average(self): + """rotate by checking each angle and guess most suitable""" + + w, h = self.image.size + pixels = self.pixels + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 155 + + highest = {} + counts = {} + + for angle in xrange(-45, 45): + + tmpimage = self.image.rotate(angle) + + pixels = tmpimage.load() + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + count = {} + + for x in xrange(w): + count[x] = 0 + for y in xrange(h): + if pixels[x, y] == 155: + count[x] += 1 + + sum = 0 + cnt = 0 + + for x in count.values(): + if x != 0: + sum += x + cnt += 1 + + avg = sum / cnt + counts[angle] = cnt + highest[angle] = 0 + for x in count.values(): + if x > highest[angle]: + highest[angle] = x + + highest[angle] = highest[angle] - avg + + hkey = 0 + hvalue = 0 + + for key, value in highest.iteritems(): + if value > hvalue: + hkey = key + hvalue = value + + self.image = self.image.rotate(hkey) + pixels = self.image.load() + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + if pixels[x, y] == 155: + pixels[x, y] = 0 + + self.pixels = pixels + + def split_captcha_letters(self): + captcha = self.image + started = False + letters = [] + width, height = captcha.size + bottomY, topY = 0, height + pixels = captcha.load() + + for x in xrange(width): + black_pixel_in_col = False + for y in xrange(height): + if pixels[x, y] != 255: + if not started: + started = True + firstX = x + lastX = x + + if y > bottomY: + bottomY = y + if y < topY: + topY = y + if x > lastX: + lastX = x + + black_pixel_in_col = True + + if black_pixel_in_col is False and started is True: + rect = (firstX, topY, lastX, bottomY) + new_captcha = captcha.crop(rect) + + w, h = new_captcha.size + if w > 5 and h > 5: + letters.append(new_captcha) + + started = False + bottomY, topY = 0, height + + return letters + + def correct(self, values, var=None): + if var: + result = var + else: + result = self.result_captcha + + for key, item in values.iteritems(): + + if key.__class__ == str: + result = result.replace(key, item) + else: + for expr in key: + result = result.replace(expr, item) + + if var: + return result + else: + self.result_captcha = result diff --git a/pyload/plugins/base/__init__.py b/pyload/plugins/base/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/pyload/plugins/base/__init__.py |