From 5499be89203a18ca61a21cfc7266cf0f4ebe6547 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 15 Dec 2011 23:18:21 +0100 Subject: refractoring --- module/plugins/AccountManager.py | 2 +- module/plugins/Base.py | 116 +++++++++++++++++++++++++++++++++++++++ module/plugins/Hoster.py | 22 ++++---- module/plugins/Plugin.py | 96 +------------------------------- module/plugins/PluginManager.py | 6 ++ 5 files changed, 136 insertions(+), 106 deletions(-) create mode 100644 module/plugins/Base.py (limited to 'module/plugins') diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index fc521d36c..4f4d9f68d 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -22,7 +22,7 @@ from shutil import copy from threading import Lock -from module.PullEvents import AccountUpdateEvent +from module.interaction.PullEvents import AccountUpdateEvent from module.utils import chmod, lock ACC_VERSION = 1 diff --git a/module/plugins/Base.py b/module/plugins/Base.py new file mode 100644 index 000000000..98573ea63 --- /dev/null +++ b/module/plugins/Base.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +class Base(object): + """ + The Base plugin class with all shared methods and every possible attribute for plugin definition. + """ + __version__ = "0.4" + #: Regexp pattern which will be matched for download plugins + __pattern__ = r"" + #: Flat config definition + __config__ = tuple() + #: Short description, one liner + __description__ = "" + #: More detailed text + __long_description__ = """""" + #: List of needed modules + __dependencies__ = tuple() + #: Tags to categorize the plugin + __tags__ = tuple() + #: Base64 encoded .png icon + __icon__ = "" + #: Alternative, link to png icon + __icon_url__ = "" + #: Url with general information/support/discussion + __url__ = "" + __author_name__ = tuple() + __author_mail__ = tuple() + + + def __init__(self, core): + self.__name__ = self.__class__.__name__ + + #: Core instance + self.core = core + #: logging instance + self.log = core.log + #: core config + self.config = core.config + + #log functions + def logInfo(self, *args): + self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logWarning(self, *args): + self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logError(self, *args): + self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + def logDebug(self, *args): + self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + + + def setConf(self, option, value): + """ see `setConfig` """ + self.core.config.setPlugin(self.__name__, option, value) + + def setConfig(self, option, value): + """ Set config value for current plugin + + :param option: + :param value: + :return: + """ + self.setConf(option, value) + + def getConf(self, option): + """ see `getConfig` """ + return self.core.config.getPlugin(self.__name__, option) + + def getConfig(self, option): + """ Returns config value for current plugin + + :param option: + :return: + """ + return self.getConf(option) + + def setStorage(self, key, value): + """ Saves a value persistently to the database """ + self.core.db.setStorage(self.__name__, key, value) + + def store(self, key, value): + """ same as `setStorage` """ + self.core.db.setStorage(self.__name__, key, value) + + def getStorage(self, key=None, default=None): + """ Retrieves saved value or dict of all saved entries if key is None """ + if key is not None: + return self.core.db.getStorage(self.__name__, key) or default + return self.core.db.getStorage(self.__name__, key) + + def retrieve(self, *args, **kwargs): + """ same as `getStorage` """ + return self.getStorage(*args, **kwargs) + + def delStorage(self, key): + """ Delete entry in db """ + self.core.db.delStorage(self.__name__, key) diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 814a70949..aa50099fb 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -19,15 +19,15 @@ from module.plugins.Plugin import Plugin -def getInfo(self): - #result = [ .. (name, size, status, url) .. ] - return - class Hoster(Plugin): - __name__ = "Hoster" - __version__ = "0.1" - __pattern__ = None - __type__ = "hoster" - __description__ = """Base hoster plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + + @staticmethod + def getInfo(urls): + """This method is used to retrieve the online status of files for hoster plugins. + It has to *yield* list of tuples with the result in this format (name, size, status, url), + where status is one of API pyfile statusses. + + :param urls: List of urls + :return: + """ + pass \ No newline at end of file diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index f7587d3f2..b3c22f983 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -29,17 +29,9 @@ if os.name != "nt": from pwd import getpwnam from grp import getgrnam -from itertools import islice - -from module.utils import save_join, save_path, fs_encode - -def chunks(iterable, size): - it = iter(iterable) - item = list(islice(it, size)) - while item: - yield item - item = list(islice(it, size)) +from Base import Base +from module.utils import save_join, save_path, fs_encode, chunks class Abort(Exception): """ raised when aborted """ @@ -61,95 +53,11 @@ class SkipDownload(Exception): """ raised when download should be skipped """ -class Base(object): - """ - A Base class with log/config/db methods *all* plugin types can use - """ - - def __init__(self, core): - #: Core instance - self.core = core - #: logging instance - self.log = core.log - #: core config - self.config = core.config - - #log functions - def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - - def setConf(self, option, value): - """ see `setConfig` """ - self.core.config.setPlugin(self.__name__, option, value) - - def setConfig(self, option, value): - """ Set config value for current plugin - - :param option: - :param value: - :return: - """ - self.setConf(option, value) - - def getConf(self, option): - """ see `getConfig` """ - return self.core.config.getPlugin(self.__name__, option) - - def getConfig(self, option): - """ Returns config value for current plugin - - :param option: - :return: - """ - return self.getConf(option) - - def setStorage(self, key, value): - """ Saves a value persistently to the database """ - self.core.db.setStorage(self.__name__, key, value) - - def store(self, key, value): - """ same as `setStorage` """ - self.core.db.setStorage(self.__name__, key, value) - - def getStorage(self, key=None, default=None): - """ Retrieves saved value or dict of all saved entries if key is None """ - if key is not None: - return self.core.db.getStorage(self.__name__, key) or default - return self.core.db.getStorage(self.__name__, key) - - def retrieve(self, *args, **kwargs): - """ same as `getStorage` """ - return self.getStorage(*args, **kwargs) - - def delStorage(self, key): - """ Delete entry in db """ - self.core.db.delStorage(self.__name__, key) - - class Plugin(Base): """ Base plugin for hoster/crypter. Overwrite `process` / `decrypt` in your subclassed plugin. """ - __name__ = "Plugin" - __version__ = "0.4" - __pattern__ = None - __type__ = "hoster" - __config__ = [("name", "type", "desc", "default")] - __description__ = """Base Plugin""" - __author_name__ = ("RaNaN", "spoob", "mkaay") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") - def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index f3f5f47bc..4bf41484a 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -29,6 +29,10 @@ from traceback import print_exc from module.lib.SafeEval import const_eval as literal_eval from module.ConfigParser import IGNORE +from namedtuple import namedtuple + +PluginTuple = namedtuple("PluginTuple", "version pattern desc long_desc deps user name module") + class PluginManager: ROOT = "module.plugins." USERROOT = "userplugins." @@ -40,6 +44,8 @@ class PluginManager: DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') + ATTR = re.compile(r'__([a-z0-9_])__\s*=\s*({|\[|\(|"|\')([^__]+)', re.M | re.I) + def __init__(self, core): self.core = core -- cgit v1.2.3 From 6153be90519d11aa5678f1fa461cba75deb5a124 Mon Sep 17 00:00:00 2001 From: Jeix Date: Sat, 17 Dec 2011 14:40:43 +0100 Subject: improve share online --- module/plugins/hoster/ShareonlineBiz.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/hoster/ShareonlineBiz.py b/module/plugins/hoster/ShareonlineBiz.py index 2a6645e86..ab7f5b249 100644 --- a/module/plugins/hoster/ShareonlineBiz.py +++ b/module/plugins/hoster/ShareonlineBiz.py @@ -13,7 +13,7 @@ from module.plugins.Plugin import chunks def getInfo(urls): - api_url_base = "http://www.share-online.biz/linkcheck/linkcheck.php" + api_url_base = "http://api.share-online.biz/linkcheck.php" for chunk in chunks(urls, 90): api_param_file = {"links": "\n".join(x.replace("http://www.share-online.biz/dl/","").rstrip("/") for x in chunk)} #api only supports old style links @@ -67,7 +67,7 @@ class ShareonlineBiz(Hoster): self.handleFree() def downloadAPIData(self): - api_url_base = "http://www.share-online.biz/linkcheck/linkcheck.php?md5=1" + api_url_base = "http://api.share-online.biz/linkcheck.php?md5=1" api_param_file = {"links": self.pyfile.url.replace("http://www.share-online.biz/dl/","")} #api only supports old style links src = self.load(api_url_base, cookies=False, post=api_param_file) @@ -116,9 +116,12 @@ class ShareonlineBiz(Hoster): self.download(download_url) - check = self.checkDownload({"invalid" : "Dieses Download-Ticket ist ungültig!"}) + check = self.checkDownload({"invalid" : "Dieses Download-Ticket ist ungültig!", + "error" : "Es ist ein unbekannter Fehler aufgetreten"}) if check == "invalid": self.retry(reason=_("Invalid download ticket")) + elif check == "error": + self.fail(reason=_("ShareOnline internal problems")) def handleAPIPremium(self): #should be working better -- cgit v1.2.3 From 4e918edba6c3808b095eab1bad137a2a8cab970d Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 17 Dec 2011 21:53:57 +0100 Subject: updated plugin api and plugin manager --- module/plugins/Base.py | 7 +- module/plugins/PluginManager.py | 185 +++++++++++++++++---------------- module/plugins/hooks/UpdateManager.py | 4 +- module/plugins/hoster/RapidshareCom.py | 2 +- 4 files changed, 103 insertions(+), 95 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 98573ea63..b166ebae8 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -17,11 +17,16 @@ @author: RaNaN """ +# TODO: config format definition +# more attributes if needed +# get rid of catpcha & container plugins ?! (move to crypter & internals) +# adapt old plugins as needed + class Base(object): """ The Base plugin class with all shared methods and every possible attribute for plugin definition. """ - __version__ = "0.4" + __version__ = "0.1" #: Regexp pattern which will be matched for download plugins __pattern__ = r"" #: Flat config definition diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index 4bf41484a..744e09ef0 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -23,28 +23,27 @@ import sys from os import listdir, makedirs from os.path import isfile, join, exists, abspath from sys import version_info -from itertools import chain +from time import time from traceback import print_exc from module.lib.SafeEval import const_eval as literal_eval from module.ConfigParser import IGNORE +from module.plugins.Base import Base from namedtuple import namedtuple -PluginTuple = namedtuple("PluginTuple", "version pattern desc long_desc deps user name module") +PluginTuple = namedtuple("PluginTuple", "version re desc long_desc deps user path") class PluginManager: ROOT = "module.plugins." USERROOT = "userplugins." TYPES = ("crypter", "container", "hoster", "captcha", "accounts", "hooks", "internal") - PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') - VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)') - CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) - DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') + SINGLE = re.compile(r'__(?P[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?[a-z0-9_]+)__\s*=\s*((?:\{|\[|"{3}).*?(?:"""|\}|\]))', re.DOTALL | re.M | re.I) def __init__(self, core): self.core = core @@ -53,12 +52,18 @@ class PluginManager: self.log = core.log self.plugins = {} + self.modules = {} # cached modules + self.names = {} # overwritten names + self.history = [] # match history to speedup parsing (type, name) self.createIndex() #register for import hook sys.meta_path.append(self) + def logDebug(self, type, plugin, msg): + self.log.debug("Plugin %s | %s: %s" % (type, plugin, msg)) + def createIndex(self): """create information for all plugins available""" @@ -70,27 +75,14 @@ class PluginManager: f = open(join("userplugins", "__init__.py"), "wb") f.close() - self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) - - self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") - self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") - self.plugins["hooks"] = self.hookPlugins = self.parse("hooks") - self.plugins["internal"] = self.internalPlugins = self.parse("internal") + a = time() + for type in self.TYPES: + self.plugins[type] = self.parse(type) - self.log.debug("created index of plugins") + self.log.debug("Created index of plugins in %.2f ms", (time() - a) * 1000 ) - def parse(self, folder, pattern=False, home={}): - """ - returns dict with information - home contains parsed plugins from module. - - { - name : {path, version, config, (pattern, re), (plugin, class)} - } - - """ + def parse(self, folder, home={}): + """ Creates a dict with PluginTuples according to parsed information """ plugins = {} if home: pfolder = join("userplugins", folder) @@ -120,55 +112,56 @@ class PluginManager: name = f[:-3] if name[-1] == ".": name = name[:-4] - version = self.VERSION.findall(content) - if version: - version = float(version[0][1]) + attrs = {} + for m in self.SINGLE.findall(content) + self.MULTI.findall(content): + #replace gettext function and eval result + attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) + if not hasattr(Base, "__%s__" % m[0]): + if m[0] != "type": #TODO remove type from all plugins, its not needed + self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) + + version = 0 + + if "version" in attrs: + try: + version = float(attrs["version"]) + except ValueError: + self.logDebug(folder, name, "Invalid version %s" % attrs["version"]) + version = 9 #TODO remove when plugins are fixed, causing update loops else: - version = 0 + self.logDebug(folder, name, "No version attribute") # home contains plugins from pyload root if home and name in home: - if home[name]["v"] >= version: + if home[name].version >= version: continue if name in IGNORE or (folder, name) in IGNORE: - continue - - plugins[name] = {} - plugins[name]["v"] = version - - module = f.replace(".pyc", "").replace(".py", "") - - # the plugin is loaded from user directory - plugins[name]["user"] = True if home else False - plugins[name]["name"] = module - - if pattern: - pattern = self.PATTERN.findall(content) - - if pattern: - pattern = pattern[0][1] - else: - pattern = "^unmachtable$" - - plugins[name]["pattern"] = pattern + continue + if "pattern" in attrs and attrs["pattern"]: try: - plugins[name]["re"] = re.compile(pattern) + plugin_re = re.compile(attrs["pattern"]) except: - self.log.error(_("%s has a invalid pattern.") % name) + self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"]) + plugin_re = None + else: plugin_re = None + + + # create plugin tuple + plugin = PluginTuple(version, plugin_re, None, None, None, True if home else False, + f.replace(".pyc", "").replace(".py", "")) + plugins[name] = plugin # internals have no config if folder == "internal": self.core.config.deleteConfig(name) continue - config = self.CONFIG.findall(content) - if config: - config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" + if "config" in attrs and attrs["config"]: + config = attrs["config"] + desc = attrs["description"] if "description" in attrs else "" if type(config[0]) == tuple: config = [list(x) for x in config] @@ -189,8 +182,7 @@ class PluginManager: self.log.error("Invalid config in %s: %s" % (name, config)) elif folder == "hooks": #force config creation - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" + desc = attrs["description"] if "description" in attrs else "" config = (["activated", "bool", "Activated", False],) try: @@ -199,39 +191,54 @@ class PluginManager: self.log.error("Invalid config in %s: %s" % (name, config)) if not home: - temp = self.parse(folder, pattern, plugins) + temp = self.parse(folder, plugins) plugins.update(temp) return plugins + def parsePlugin(self): + pass + def parseUrls(self, urls): """parse plugins for given list of urls""" - last = None res = [] # tupels of (url, plugin) for url in urls: if type(url) not in (str, unicode, buffer): continue found = False - if last and last[1]["re"].match(url): - res.append((url, last[0])) + for ptype, name in self.history: + if self.plugins[ptype][name].re.match(url): + res.append((url, name)) + found = (ptype, name) + + if found: + # found match, update history + self.history.remove(found) + self.history.insert(0, found) continue - for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(), - self.containerPlugins.iteritems()): - if value["re"].match(url): - res.append((url, name)) - last = (name, value) - found = True - break + for ptype in ("crypter", "hoster", "container"): + for name, plugin in self.plugins[ptype].iteritems(): + if plugin.re.match(url): + res.append((url, name)) + self.history.insert(0, (ptype, name)) + del self.history[10:] # cut down to size of 10 + found = True + break if not found: res.append((url, "BasePlugin")) return res + def getPlugins(self, type): + # TODO clean this workaround + if type not in self.plugins: type+="s" # append s, so updater can find the plugins + return self.plugins[type] + def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): for ptype in pluginlist: if name in self.plugins[ptype]: @@ -244,7 +251,7 @@ class PluginManager: if not plugin: self.log.warning("Plugin %s not found." % name) - plugin = self.hosterPlugins["BasePlugin"] + plugin = self.plugins["hoster"]["BasePlugin"] if "new_module" in plugin and not original: return plugin["new_module"] @@ -268,11 +275,11 @@ class PluginManager: """ plugins = self.plugins[type] if name in plugins: - if "module" in plugins[name]: return plugins[name]["module"] + if (type, name) in self.modules: return self.modules[(type, name)] try: - module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]["name"]), globals(), locals(), - plugins[name]["name"]) - plugins[name]["module"] = module #cache import, maybe unneeded + module = __import__(self.ROOT + "%s.%s" % (type, plugins[name].path), globals(), locals(), + plugins[name].path) + self.modules[(type, name)] = module # cache import, maybne unneeded return module except Exception, e: self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) @@ -286,7 +293,7 @@ class PluginManager: def getAccountPlugins(self): """return list of account plugin names""" - return self.accountPlugins.keys() + return self.plugins["accounts"].keys() def find_module(self, fullname, path=None): #redirecting imports if necesarry @@ -300,10 +307,10 @@ class PluginManager: if type in self.plugins and name in self.plugins[type]: #userplugin is a newer version - if not user and self.plugins[type][name]["user"]: + if not user and self.plugins[type][name].user: return self - #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name]["user"]: + #imported from userdir, but pyloads is newer + if user and not self.plugins[type][name].user: return self @@ -335,7 +342,7 @@ class PluginManager: self.log.debug("Request reload of plugins: %s" % type_plugins) as_dict = {} - for t,n in type_plugins: + for t, n in type_plugins: if t in as_dict: as_dict[t].append(n) else: @@ -348,16 +355,13 @@ class PluginManager: for type in as_dict.iterkeys(): for plugin in as_dict[type]: if plugin in self.plugins[type]: - if "module" in self.plugins[type][plugin]: + if (type, plugin) in self.modules: self.log.debug("Reloading %s" % plugin) - reload(self.plugins[type][plugin]["module"]) + reload(self.modules[(type, plugin)]) - #index creation - self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) - self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") - self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") + # index re-creation + for type in ("crypter", "container", "hoster", "captcha", "accounts"): + self.plugins[type] = self.parse(type) if "accounts" in as_dict: #accounts needs to be reloaded self.core.accountManager.initPlugins() @@ -366,7 +370,6 @@ class PluginManager: return True - if __name__ == "__main__": _ = lambda x: x pypath = "/home/christian/Projekte/pyload-0.4/module/plugins" diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py index 920a88060..8cbc56c42 100644 --- a/module/plugins/hooks/UpdateManager.py +++ b/module/plugins/hooks/UpdateManager.py @@ -129,10 +129,10 @@ class UpdateManager(Hook): else: type = prefix - plugins = getattr(self.core.pluginManager, "%sPlugins" % type) + plugins = self.core.pluginManager.getPlugins(type) if name in plugins: - if float(plugins[name]["v"]) >= float(version): + if float(plugins[name].version) >= float(version): continue if name in IGNORE or (type, name) in IGNORE: diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py index 0d927c525..f3011a488 100644 --- a/module/plugins/hoster/RapidshareCom.py +++ b/module/plugins/hoster/RapidshareCom.py @@ -52,7 +52,7 @@ class RapidshareCom(Hoster): __pattern__ = r"https?://[\w\.]*?rapidshare.com/(?:files/(?P\d*?)/(?P[^?]+)|#!download\|(?:\w+)\|(?P\d+)\|(?P[^|]+))" __version__ = "1.37" __description__ = """Rapidshare.com Download Hoster""" - __config__ = [["server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None"]] + __config__ = [("server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None")] __author_name__ = ("spoob", "RaNaN", "mkaay") __author_mail__ = ("spoob@pyload.org", "ranan@pyload.org", "mkaay@mkaay.de") -- cgit v1.2.3 From de5e99f3c3082b96af398c01aec49e231726a36b Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 17 Dec 2011 23:55:01 +0100 Subject: pluginmanager cleanup --- module/plugins/PluginManager.py | 199 +++++++++++++++++++++++----------------- 1 file changed, 116 insertions(+), 83 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index 744e09ef0..b1a82bcb6 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -21,7 +21,7 @@ import re import sys from os import listdir, makedirs -from os.path import isfile, join, exists, abspath +from os.path import isfile, join, exists, abspath, basename from sys import version_info from time import time from traceback import print_exc @@ -66,7 +66,7 @@ class PluginManager: def createIndex(self): """create information for all plugins available""" - + # add to path, so we can import from userplugins sys.path.append(abspath("")) if not exists("userplugins"): @@ -79,10 +79,10 @@ class PluginManager: for type in self.TYPES: self.plugins[type] = self.parse(type) - self.log.debug("Created index of plugins in %.2f ms", (time() - a) * 1000 ) + self.log.debug("Created index of plugins in %.2f ms", (time() - a) * 1000) - def parse(self, folder, home={}): - """ Creates a dict with PluginTuples according to parsed information """ + def parse(self, folder, home=None): + """ Analyze and parses all plugins in folder """ plugins = {} if home: pfolder = join("userplugins", folder) @@ -98,10 +98,6 @@ class PluginManager: for f in listdir(pfolder): if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - data = open(join(pfolder, f)) - content = data.read() - data.close() - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): continue elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): @@ -109,95 +105,111 @@ class PluginManager: elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): continue + # replace suffix and version tag name = f[:-3] if name[-1] == ".": name = name[:-4] - attrs = {} - for m in self.SINGLE.findall(content) + self.MULTI.findall(content): - #replace gettext function and eval result - attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) - if not hasattr(Base, "__%s__" % m[0]): - if m[0] != "type": #TODO remove type from all plugins, its not needed - self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) - - version = 0 - - if "version" in attrs: - try: - version = float(attrs["version"]) - except ValueError: - self.logDebug(folder, name, "Invalid version %s" % attrs["version"]) - version = 9 #TODO remove when plugins are fixed, causing update loops - else: - self.logDebug(folder, name, "No version attribute") + plugin = self.parsePlugin(join(pfolder, f), folder, name, home) + if plugin: + plugins[name] = plugin - # home contains plugins from pyload root - if home and name in home: - if home[name].version >= version: - continue + if not home: + temp = self.parse(folder, plugins) + plugins.update(temp) - if name in IGNORE or (folder, name) in IGNORE: - continue + return plugins - if "pattern" in attrs and attrs["pattern"]: - try: - plugin_re = re.compile(attrs["pattern"]) - except: - self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"]) - plugin_re = None - else: plugin_re = None + def parsePlugin(self, filename, folder, name, home=None): + """ Parses a plugin from disk, folder means plugin type in this context. Also sets config. + :arg home: dict with plugins, of which the found one will be matched against (according version) + :returns PluginTuple""" - # create plugin tuple - plugin = PluginTuple(version, plugin_re, None, None, None, True if home else False, - f.replace(".pyc", "").replace(".py", "")) + data = open(filename, "rb") + content = data.read() + data.close() - plugins[name] = plugin + attrs = {} + for m in self.SINGLE.findall(content) + self.MULTI.findall(content): + #replace gettext function and eval result + try: + attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) + except: + self.logDebug(folder, name, "Error when parsing: %s" % m[-1]) + return + if not hasattr(Base, "__%s__" % m[0]): + if m[0] != "type": #TODO remove type from all plugins, its not needed + self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) - # internals have no config - if folder == "internal": - self.core.config.deleteConfig(name) - continue + version = 0 - if "config" in attrs and attrs["config"]: - config = attrs["config"] - desc = attrs["description"] if "description" in attrs else "" + if "version" in attrs: + try: + version = float(attrs["version"]) + except ValueError: + self.logDebug(folder, name, "Invalid version %s" % attrs["version"]) + version = 9 #TODO remove when plugins are fixed, causing update loops + else: + self.logDebug(folder, name, "No version attribute") - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] + # home contains plugins from pyload root + if home and name in home: + if home[name].version >= version: + return - if folder == "hooks": - append = True - for item in config: - if item[0] == "activated": append = False + if name in IGNORE or (folder, name) in IGNORE: + return - # activated flag missing - if append: config.append(["activated", "bool", "Activated", False]) + if "pattern" in attrs and attrs["pattern"]: + try: + plugin_re = re.compile(attrs["pattern"]) + except: + self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"]) + plugin_re = None + else: plugin_re = None - try: - self.core.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) - elif folder == "hooks": #force config creation - desc = attrs["description"] if "description" in attrs else "" - config = (["activated", "bool", "Activated", False],) + # create plugin tuple + plugin = PluginTuple(version, plugin_re, None, None, None, bool(home), filename) - try: - self.core.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) - if not home: - temp = self.parse(folder, plugins) - plugins.update(temp) + # internals have no config + if folder == "internal": + self.core.config.deleteConfig(name) + return plugin - return plugins + if "config" in attrs and attrs["config"]: + config = attrs["config"] + desc = attrs["description"] if "description" in attrs else "" - def parsePlugin(self): - pass + if type(config[0]) == tuple: + config = [list(x) for x in config] + else: + config = [list(config)] + + if folder == "hooks": + append = True + for item in config: + if item[0] == "activated": append = False + + # activated flag missing + if append: config.append(["activated", "bool", "Activated", False]) + + try: + self.core.config.addPluginConfig(name, config, desc) + except: + self.log.error("Invalid config in %s: %s" % (name, config)) + + elif folder == "hooks": #force config creation + desc = attrs["description"] if "description" in attrs else "" + config = (["activated", "bool", "Activated", False],) + + try: + self.core.config.addPluginConfig(name, config, desc) + except: + self.log.error("Invalid config in %s: %s" % (name, config)) + + return plugin def parseUrls(self, urls): @@ -206,7 +218,9 @@ class PluginManager: res = [] # tupels of (url, plugin) for url in urls: - if type(url) not in (str, unicode, buffer): continue + if type(url) not in (str, unicode, buffer): + self.log.debug("Parsing invalid type %s" % type(url)) + continue found = False for ptype, name in self.history: @@ -214,7 +228,7 @@ class PluginManager: res.append((url, name)) found = (ptype, name) - if found: + if found and self.history[0] != found: # found match, update history self.history.remove(found) self.history.insert(0, found) @@ -236,7 +250,7 @@ class PluginManager: def getPlugins(self, type): # TODO clean this workaround - if type not in self.plugins: type+="s" # append s, so updater can find the plugins + if type not in self.plugins: type += "s" # append s, so updater can find the plugins return self.plugins[type] def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): @@ -277,8 +291,9 @@ class PluginManager: if name in plugins: if (type, name) in self.modules: return self.modules[(type, name)] try: - module = __import__(self.ROOT + "%s.%s" % (type, plugins[name].path), globals(), locals(), - plugins[name].path) + # convert path to python recognizable import + path = basename(plugins[name].path).replace(".pyc", "").replace(".py", "") + module = __import__(self.ROOT + "%s.%s" % (type, path), globals(), locals(), path) self.modules[(type, name)] = module # cache import, maybne unneeded return module except Exception, e: @@ -369,6 +384,24 @@ class PluginManager: return True + def loadIcons(self): + """Loads all icons from plugins, plugin type is not in result, because its not important here. + + :return: Dict of names mapped to icons + """ + pass + + def loadIcon(self, type, name): + """ load icon for single plugin, base64 encoded""" + pass + + def checkDependencies(self, type, name): + """ Check deps for given plugin + + :return: List of unfullfilled dependencies + """ + pass + if __name__ == "__main__": _ = lambda x: x -- cgit v1.2.3 From 6eae782f13953dd0ba2bbe1b582cf33fd4d7d90a Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 19 Dec 2011 23:10:49 +0100 Subject: configparser v2, warning CONFIG will be DELETED. --- module/plugins/Base.py | 4 ++-- module/plugins/Hook.py | 2 +- module/plugins/Plugin.py | 2 +- module/plugins/PluginManager.py | 41 +++++++++++++++++++---------------- module/plugins/hooks/UpdateManager.py | 2 +- 5 files changed, 27 insertions(+), 24 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Base.py b/module/plugins/Base.py index b166ebae8..5a78fddc3 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -75,7 +75,7 @@ class Base(object): def setConf(self, option, value): """ see `setConfig` """ - self.core.config.setPlugin(self.__name__, option, value) + self.core.config.set(self.__name__, option, value) def setConfig(self, option, value): """ Set config value for current plugin @@ -88,7 +88,7 @@ class Base(object): def getConf(self, option): """ see `getConfig` """ - return self.core.config.getPlugin(self.__name__, option) + return self.core.config.get(self.__name__, option) def getConfig(self, option): """ Returns config value for current plugin diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index 5efd08bae..860dc76bb 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -119,7 +119,7 @@ class Hook(Base): def isActivated(self): """ checks if hook is activated""" - return self.config.getPlugin(self.__name__, "activated") + return self.config.get(self.__name__, "activated") #event methods - overwrite these if needed diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index b3c22f983..327e717a0 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -258,7 +258,7 @@ class Plugin(Base): temp_file.write(img) temp_file.close() - has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins + has_plugin = self.__name__ in self.core.pluginManager.getPlugins("captcha") if self.core.captcha: Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index b1a82bcb6..9845590cf 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -27,12 +27,17 @@ from time import time from traceback import print_exc from module.lib.SafeEval import const_eval as literal_eval -from module.ConfigParser import IGNORE from module.plugins.Base import Base -from namedtuple import namedtuple +from new_collections import namedtuple -PluginTuple = namedtuple("PluginTuple", "version re desc long_desc deps user path") +# ignore these plugin configs, mainly because plugins were wiped out +IGNORE = ( + "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('hooks', 'UnRar'), + 'EasyShareCom', 'FlyshareCz' + ) + +PluginTuple = namedtuple("PluginTuple", "version re deps user path") class PluginManager: ROOT = "module.plugins." @@ -57,6 +62,9 @@ class PluginManager: self.history = [] # match history to speedup parsing (type, name) self.createIndex() + + self.core.config.parseValues(self.core.config.PLUGIN) + #register for import hook sys.meta_path.append(self) @@ -168,19 +176,23 @@ class PluginManager: plugin_re = None else: plugin_re = None + deps = attrs.get("dependencies", None) # create plugin tuple - plugin = PluginTuple(version, plugin_re, None, None, None, bool(home), filename) + plugin = PluginTuple(version, plugin_re, deps, bool(home), filename) # internals have no config if folder == "internal": - self.core.config.deleteConfig(name) return plugin + if folder == "hooks" and "config" not in attrs: + attrs["config"] = (["activated", "bool", "Activated", False],) + if "config" in attrs and attrs["config"]: config = attrs["config"] - desc = attrs["description"] if "description" in attrs else "" + desc = attrs.get("description", "") + long_desc = attrs.get("long_description", "") if type(config[0]) == tuple: config = [list(x) for x in config] @@ -193,21 +205,12 @@ class PluginManager: if item[0] == "activated": append = False # activated flag missing - if append: config.append(["activated", "bool", "Activated", False]) - - try: - self.core.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) - - elif folder == "hooks": #force config creation - desc = attrs["description"] if "description" in attrs else "" - config = (["activated", "bool", "Activated", False],) + if append: config.insert(0, ("activated", "bool", "Activated", False)) try: - self.core.config.addPluginConfig(name, config, desc) + self.core.config.addConfigSection(name, name, desc, long_desc, config) except: - self.log.error("Invalid config in %s: %s" % (name, config)) + self.logDebug(folder, name, "Invalid config %s" % config) return plugin @@ -294,7 +297,7 @@ class PluginManager: # convert path to python recognizable import path = basename(plugins[name].path).replace(".pyc", "").replace(".py", "") module = __import__(self.ROOT + "%s.%s" % (type, path), globals(), locals(), path) - self.modules[(type, name)] = module # cache import, maybne unneeded + self.modules[(type, name)] = module # cache import, maybe unneeded return module except Exception, e: self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py index 8cbc56c42..4324a96ba 100644 --- a/module/plugins/hooks/UpdateManager.py +++ b/module/plugins/hooks/UpdateManager.py @@ -24,7 +24,7 @@ from os import stat from os.path import join, exists from time import time -from module.ConfigParser import IGNORE +from module.plugins.PluginManager import IGNORE from module.network.RequestFactory import getURL from module.plugins.Hook import threaded, Expose, Hook -- cgit v1.2.3 From 958bf611f5d9d117f19f824990ec6fd6b537e967 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 22 Dec 2011 23:45:38 +0100 Subject: accountmanager v2, delete your accounts.conf and re-enter them in pyload, new nice debug functions, try core.shell() and core.breakpoint() --- module/plugins/Account.py | 335 ++++++++++++++------------------- module/plugins/AccountManager.py | 208 ++++++++------------ module/plugins/Base.py | 10 + module/plugins/Plugin.py | 14 +- module/plugins/PluginManager.py | 29 ++- module/plugins/internal/MultiHoster.py | 21 +-- 6 files changed, 266 insertions(+), 351 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Account.py b/module/plugins/Account.py index c147404e0..9da8d0357 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -1,63 +1,72 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from random import choice from time import time from traceback import print_exc from threading import RLock from Plugin import Base from module.utils import compare_time, parseFileSize, lock +from module.config.converter import from_string +from module.Api import AccountInfo +from module.network.CookieJar import CookieJar class WrongPassword(Exception): pass - -class Account(Base): +#noinspection PyUnresolvedReferences +class Account(Base, AccountInfo): """ Base class for every Account plugin. Just overwrite `login` and cookies will be stored and account becomes accessible in\ associated hoster plugin. Plugin should also provide `loadAccountInfo` """ - __name__ = "Account" - __version__ = "0.2" - __type__ = "account" - __description__ = """Account Plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + + # Default values + valid = True + validuntil = None + trafficleft = None + maxtraffic = None + premium = True + activated = True #: after that time [in minutes] pyload will relogin the account login_timeout = 600 #: account data will be reloaded after this time info_threshold = 600 + # known options + known_opt = ["time", "limitDL"] - def __init__(self, manager, accounts): + + def __init__(self, manager, loginname, password, options): Base.__init__(self, manager.core) + if "activated" in options: + activated = from_string(options["activated"], "bool") + else: + activated = Account.activated + + for opt in self.known_opt: + if opt not in options: + options[opt] = "" + + for opt in options.keys(): + if opt not in self.known_opt: + del options[opt] + + # default account attributes + AccountInfo.__init__(self, self.__name__, loginname, Account.valid, Account.validuntil, Account.trafficleft, + Account.maxtraffic, Account.premium, activated, options) + self.manager = manager - self.accounts = {} - self.infos = {} # cache for account information + self.lock = RLock() + self.timestamp = 0 + self.login_ts = 0 # timestamp for login + self.cj = CookieJar(self.__name__) + self.password = password + self.error = None - self.timestamps = {} - self.setAccounts(accounts) self.init() def init(self): @@ -70,76 +79,68 @@ class Account(Base): :param data: data dictionary :param req: `Request` instance """ - pass + raise NotImplemented @lock - def _login(self, user, data): + def _login(self): # set timestamp for login - self.timestamps[user] = time() - - req = self.getAccountRequest(user) + self.login_ts = time() + + req = self.getAccountRequest() try: - self.login(user, data, req) + self.login(self.loginname, {"password": self.password}, req) + self.valid = True except WrongPassword: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": _("Wrong Password")}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname + , "msg": _("Wrong Password")}) + self.valid = False except Exception, e: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": e}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname + , "msg": e}) + self.valid = False if self.core.debug: print_exc() finally: - if req: req.close() - - def relogin(self, user): - req = self.getAccountRequest(user) - if req: - req.cj.clear() req.close() - if user in self.infos: - del self.infos[user] #delete old information - - self._login(user, self.accounts[user]) - def setAccounts(self, accounts): - self.accounts = accounts - for user, data in self.accounts.iteritems(): - self._login(user, data) - self.infos[user] = {} + def restoreDefaults(self): + self.valid = Account.valid + self.validuntil = Account.validuntil + self.trafficleft = Account.trafficleft + self.maxtraffic = Account.maxtraffic + self.premium = Account.premium + self.activated = Account.activated - def updateAccounts(self, user, password=None, options={}): + def update(self, password=None, options={}): """ updates account and return true if anything changed """ - if user in self.accounts: - self.accounts[user]["valid"] = True #do not remove or accounts will not login - if password: - self.accounts[user]["password"] = password - self.relogin(user) - return True - if options: - before = self.accounts[user]["options"] - self.accounts[user]["options"].update(options) - return self.accounts[user]["options"] != before - else: - self.accounts[user] = {"password": password, "options": options, "valid": True} - self._login(user, self.accounts[user]) + self.login_ts = 0 + + if "activated" in options: + self.activated = from_string(options["avtivated"], "bool") + + if password: + self.password = password + self._login() return True + if options: + # remove unknown options + for opt in options.keys(): + if opt not in self.known_opt: + del options[opt] - def removeAccount(self, user): - if user in self.accounts: - del self.accounts[user] - if user in self.infos: - del self.infos[user] - if user in self.timestamps: - del self.timestamps[user] + before = self.options + self.options.update(options) + return self.options != before + + def getAccountRequest(self): + return self.core.requestFactory.getRequest(self.__name__, self.cj) @lock - def getAccountInfo(self, name, force=False): + def getAccountInfo(self, force=False): """retrieve account infos for an user, do **not** overwrite this method!\\ just use it to retrieve infos in hoster plugins. see `loadAccountInfo` @@ -147,113 +148,55 @@ class Account(Base): :param force: reloads cached account information :return: dictionary with information """ - data = Account.loadAccountInfo(self, name) + if force or self.timestamp + self.info_threshold * 60 < time(): - if force or name not in self.infos: - self.logDebug("Get Account Info for %s" % name) - req = self.getAccountRequest(name) + # make sure to login + self.checkLogin() + self.logDebug("Get Account Info for %s" % self.loginname) + req = self.getAccountRequest() try: - infos = self.loadAccountInfo(name, req) - if not type(infos) == dict: - raise Exception("Wrong return format") + infos = self.loadAccountInfo(self.loginname, req) except Exception, e: infos = {"error": str(e)} - if req: req.close() + req.close() self.logDebug("Account Info: %s" % str(infos)) + self.timestamp = time() + + self.restoreDefaults() # reset to initial state + if type(infos) == dict: # copy result from dict to class + for k, v in infos.iteritems(): + if hasattr(self, k): + setattr(self, k, v) + else: + self.logDebug("Unknown attribute %s=%s" % (k, v)) + + def isPremium(self, user=None): + if user: self.logDebug("Deprecated Argument user for .isPremium()", user) + return self.premium + + def isUsable(self): + """Check several contraints to determine if account should be used""" + if not self.valid or not self.activated: return False + + if self.options["time"]: + time_data = "" + try: + time_data = self.options["time"] + start, end = time_data.split("-") + if not compare_time(start.split(":"), end.split(":")): + return False + except: + self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + + if 0 < self.validuntil < time(): + return False + if self.trafficleft is 0: # test explicity for 0 + return False - infos["timestamp"] = time() - self.infos[name] = infos - elif "timestamp" in self.infos[name] and self.infos[name][ - "timestamp"] + self.info_threshold * 60 < time(): - self.logDebug("Reached timeout for account data") - self.scheduleRefresh(name) - - data.update(self.infos[name]) - return data - - def isPremium(self, user): - info = self.getAccountInfo(user) - return info["premium"] - - def loadAccountInfo(self, name, req=None): - """this should be overwritten in account plugin,\ - and retrieving account information for user - - :param name: - :param req: `Request` instance - :return: - """ - return { - "validuntil": None, # -1 for unlimited - "login": name, - #"password": self.accounts[name]["password"], #@XXX: security - "options": self.accounts[name]["options"], - "valid": self.accounts[name]["valid"], - "trafficleft": None, # in kb, -1 for unlimited - "maxtraffic": None, - "premium": True, #useful for free accounts - "timestamp": 0, #time this info was retrieved - "type": self.__name__, - } - - def getAllAccounts(self, force=False): - return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()] - - def getAccountRequest(self, user=None): - if not user: - user, data = self.selectAccount() - if not user: - return None - - req = self.core.requestFactory.getRequest(self.__name__, user) - return req - - def getAccountCookies(self, user=None): - if not user: - user, data = self.selectAccount() - if not user: - return None - - cj = self.core.requestFactory.getCookieJar(self.__name__, user) - return cj - - def getAccountData(self, user): - return self.accounts[user] - - def selectAccount(self): - """ returns an valid account name and data""" - usable = [] - for user, data in self.accounts.iteritems(): - if not data["valid"]: continue - - if "time" in data["options"] and data["options"]["time"]: - time_data = "" - try: - time_data = data["options"]["time"][0] - start, end = time_data.split("-") - if not compare_time(start.split(":"), end.split(":")): - continue - except: - self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) - - if user in self.infos: - if "validuntil" in self.infos[user]: - if self.infos[user]["validuntil"] > 0 and time() > self.infos[user]["validuntil"]: - continue - if "trafficleft" in self.infos[user]: - if self.infos[user]["trafficleft"] == 0: - continue - - usable.append((user, data)) - - if not usable: return None, None - return choice(usable) - - def canUse(self): - return False if self.selectAccount() == (None, None) else True + return True def parseTraffic(self, string): #returns kbyte return parseFileSize(string) / 1024 @@ -261,32 +204,36 @@ class Account(Base): def wrongPassword(self): raise WrongPassword - def empty(self, user): - if user in self.infos: - self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user) + def empty(self, user=None): + if user: self.logDebug("Deprecated argument user for .empty()", user) + + self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % self.login) - self.infos[user].update({"trafficleft": 0}) - self.scheduleRefresh(user, 30 * 60) + self.trafficleft = 0 + self.scheduleRefresh(30 * 60) def expired(self, user): if user in self.infos: self.logWarning(_("Account %s is expired, checking again in 1h") % user) - self.infos[user].update({"validuntil": time() - 1}) - self.scheduleRefresh(user, 60 * 60) + self.validuntil = time() - 1 + self.scheduleRefresh(60 * 60) - def scheduleRefresh(self, user, time=0, force=True): + def scheduleRefresh(self, time=0, force=True): """ add task to refresh account info to sheduler """ - self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) - self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) + self.logDebug("Scheduled Account refresh for %s in %s seconds." % (self.loginname, time)) + self.core.scheduler.addJob(time, self.getAccountInfo, [force]) @lock - def checkLogin(self, user): + def checkLogin(self): """ checks if user is still logged in """ - if user in self.timestamps: - if self.timestamps[user] + self.login_timeout * 60 < time(): - self.logDebug("Reached login timeout for %s" % user) - self.relogin(user) - return False + if self.login_ts + self.login_timeout * 60 < time(): + if self.login_ts: # seperate from fresh login to have better debug logs + self.logDebug("Reached login timeout for %s" % self.loginname) + else: + self.logDebug("Login with %s" % self.loginname) + + self._login() + return False return True diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index 4f4d9f68d..c718510ed 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -17,169 +17,119 @@ @author: RaNaN """ -from os.path import exists -from shutil import copy - from threading import Lock +from random import choice +from module.common.json_layer import json from module.interaction.PullEvents import AccountUpdateEvent -from module.utils import chmod, lock - -ACC_VERSION = 1 +from module.utils import lock class AccountManager(): """manages all accounts""" - #---------------------------------------------------------------------- def __init__(self, core): """Constructor""" self.core = core self.lock = Lock() - self.initPlugins() - self.saveAccounts() # save to add categories to conf + self.loadAccounts() - def initPlugins(self): - self.accounts = {} # key = ( plugin ) - self.plugins = {} + def loadAccounts(self): + """loads all accounts available""" - self.initAccountPlugins() - self.loadAccounts() + self.accounts = {} + for plugin, loginname, activated, password, options in self.core.db.loadAccounts(): + # put into options as used in other context + options = json.loads(options) if options else {} + options["activated"] = activated - def getAccountPlugin(self, plugin): - """get account instance for plugin or None if anonymous""" - if plugin in self.accounts: - if plugin not in self.plugins: - self.plugins[plugin] = self.core.pluginManager.loadClass("accounts", plugin)(self, self.accounts[plugin]) + self.createAccount(plugin, loginname, password, options) + + return - return self.plugins[plugin] - else: - return None - - def getAccountPlugins(self): - """ get all account instances""" - - plugins = [] - for plugin in self.accounts.keys(): - plugins.append(self.getAccountPlugin(plugin)) - - return plugins - #---------------------------------------------------------------------- - def loadAccounts(self): - """loads all accounts available""" - - if not exists("accounts.conf"): - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - - f = open("accounts.conf", "rb") - content = f.readlines() - version = content[0].split(":")[1].strip() if content else "" - f.close() - - if not version or int(version) < ACC_VERSION: - copy("accounts.conf", "accounts.backup") - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - self.core.log.warning(_("Account settings deleted, due to new config format.")) - return - - - - plugin = "" - name = "" - - for line in content[1:]: - line = line.strip() - - if not line: continue - if line.startswith("#"): continue - if line.startswith("version"): continue - - if line.endswith(":") and line.count(":") == 1: - plugin = line[:-1] - self.accounts[plugin] = {} - - elif line.startswith("@"): - try: - option = line[1:].split() - self.accounts[plugin][name]["options"][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) - except: - pass - - elif ":" in line: - name, sep, pw = line.partition(":") - self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} - #---------------------------------------------------------------------- def saveAccounts(self): """save all account information""" - - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION) + "\n") - - for plugin, accounts in self.accounts.iteritems(): - f.write("\n") - f.write(plugin+":\n") - - for name,data in accounts.iteritems(): - f.write("\n\t%s:%s\n" % (name,data["password"]) ) - if data["options"]: - for option, values in data["options"].iteritems(): - f.write("\t@%s %s\n" % (option, " ".join(values))) - - f.close() - chmod(f.name, 0600) - - - #---------------------------------------------------------------------- - def initAccountPlugins(self): - """init names""" - for name in self.core.pluginManager.getAccountPlugins(): - self.accounts[name] = {} - + + data = [] + for name, plugin in self.accounts.iteritems(): + data.extend([(name, acc.loginname, acc.activated, acc.password, json.dumps(acc.options)) for acc in + plugin.itervalues()]) + self.core.db.saveAccounts(data) + + def createAccount(self, plugin, loginname, password, options): + klass = self.core.pluginManager.loadClass("accounts", plugin) + if not klass: + self.core.log.warning(_("Unknown account plugin %s") % plugin) + return + + if plugin not in self.accounts: + self.accounts[plugin] = {} + + self.core.log.debug("Create account %s:%s" % (plugin, loginname)) + + self.accounts[plugin][loginname] = klass(self, loginname, password, options) + + @lock - def updateAccount(self, plugin , user, password=None, options={}): + def updateAccount(self, plugin, user, password=None, options={}): """add or update account""" - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - updated = p.updateAccounts(user, password, options) - #since accounts is a ref in plugin self.accounts doesnt need to be updated here - + if plugin in self.accounts and user in self.accounts[plugin]: + acc = self.accounts[plugin][user] + updated = acc.update(password, options) + self.saveAccounts() - if updated: p.scheduleRefresh(user, force=False) - + if updated: acc.scheduleRefresh(force=True) + else: + self.createAccount(plugin, user, password, options) + self.saveAccounts() + @lock def removeAccount(self, plugin, user): """remove account""" - + if plugin in self.accounts and user in self.accounts[plugin]: + del self.accounts[plugin][user] + self.core.db.removeAccount(plugin, user) + else: + self.core.log.debug("Remove non existing account %s %s" % (plugin, user)) + + + @lock + def getAccountForPlugin(self, plugin): if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - p.removeAccount(user) + accs = [x for x in self.accounts[plugin].values() if x.isUsable()] + if accs: return choice(accs) - self.saveAccounts() + return None @lock - def getAccountInfos(self, force=True, refresh=False): - data = {} + def getAllAccounts(self, refresh=False): + """ Return account info, refresh afterwards if needed + :param refresh: + :return: + """ if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - force = False - - for p in self.accounts.keys(): - if self.accounts[p]: - p = self.getAccountPlugin(p) - data[p.__name__] = p.getAllAccounts(force) - else: - data[p] = [] + self.core.scheduler.addJob(0, self.core.accountManager.getAllAccounts) + + # load unavailable account info + for p_dict in self.accounts.itervalues(): + for acc in p_dict.itervalues(): + acc.getAccountInfo() + e = AccountUpdateEvent() self.core.pullManager.addEvent(e) - return data - + + return self.accounts + + def refreshAllAccounts(self): + """ Force a refresh of every account """ + for p in self.accounts.itervalues(): + for acc in p.itervalues(): + acc.getAccountInfo(True) + + def sendChange(self): e = AccountUpdateEvent() self.core.pullManager.addEvent(e) diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 5a78fddc3..36df7e423 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -17,6 +17,8 @@ @author: RaNaN """ +import sys + # TODO: config format definition # more attributes if needed # get rid of catpcha & container plugins ?! (move to crypter & internals) @@ -119,3 +121,11 @@ class Base(object): def delStorage(self, key): """ Delete entry in db """ self.core.db.delStorage(self.__name__, key) + + def shell(self): + """ open ipython shell """ + if self.core.debug: + from IPython import embed + #noinspection PyUnresolvedReferences + sys.stdout = sys._stdout + embed() diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 327e717a0..59d0f46f2 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -75,26 +75,26 @@ class Plugin(Base): self.ocr = None #captcha reader instance #: account handler instance, see :py:class:`Account` - self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + self.account = self.core.accountManager.getAccountForPlugin(self.__name__) #: premium status self.premium = False #: username/login self.user = None - if self.account and not self.account.canUse(): self.account = None + if self.account and not self.account.isUsable(): self.account = None if self.account: - self.user, data = self.account.selectAccount() + self.user, data = self.account.loginname, {} #TODO change plugins to not use data anymore #: Browser instance, see `network.Browser` - self.req = self.account.getAccountRequest(self.user) + self.req = self.account.getAccountRequest() self.chunkLimit = -1 # chunk limit, -1 for unlimited #: enables resume (will be ignored if server dont accept chunks) self.resumeDownload = True self.multiDL = True #every hoster with account should provide multiple downloads #: premium status - self.premium = self.account.isPremium(self.user) + self.premium = self.account.isPremium() else: - self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) + self.req = self.core.requestFactory.getRequest(self.__name__) #: associated pyfile instance, see `PyFile` self.pyfile = pyfile @@ -134,7 +134,7 @@ class Plugin(Base): self.thread = thread if self.account: - self.account.checkLogin(self.user) + self.account.checkLogin() else: self.req.clearCookies() diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index 9845590cf..5c31930a5 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -259,28 +259,28 @@ class PluginManager: def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): for ptype in pluginlist: if name in self.plugins[ptype]: - return self.plugins[ptype][name], ptype + return ptype, self.plugins[ptype][name] return None, None def getPlugin(self, name, original=False): """return plugin module from hoster|decrypter|container""" - plugin, type = self.findPlugin(name) + type, plugin = self.findPlugin(name) if not plugin: self.log.warning("Plugin %s not found." % name) - plugin = self.plugins["hoster"]["BasePlugin"] + name = "BasePlugin" - if "new_module" in plugin and not original: - return plugin["new_module"] + if (type, name) in self.modules and not original: + return self.modules[(type, name)] return self.loadModule(type, name) def getPluginName(self, name): """ used to obtain new name if other plugin was injected""" - plugin, type = self.findPlugin(name) + type, plugin = self.findPlugin(name) - if "new_name" in plugin: - return plugin["new_name"] + if (type, name) in self.names: + return self.names[(type, name)] return name @@ -309,10 +309,23 @@ class PluginManager: module = self.loadModule(type, name) if module: return getattr(module, name) + def hasAccountPlugin(self, plugin): + return plugin in self.plugins["accounts"] + def getAccountPlugins(self): """return list of account plugin names""" return self.plugins["accounts"].keys() + def injectPlugin(self, type, name, module, new_name): + self.modules[(type, name)] = module + self.names[(type, name)] = new_name + + def restoreState(self, type, name): + if (type, name) in self.modules: + del self.modules[(type, name)] + if (type, name) in self.names: + del self.names[(type, name)] + def find_module(self, fullname, path=None): #redirecting imports if necesarry if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index d50df3943..872e0b770 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -5,6 +5,7 @@ import re from module.utils import remove_chars from module.plugins.Hook import Hook +from module.plugins.PluginManager import PluginTuple class MultiHoster(Hook): """ @@ -43,7 +44,7 @@ class MultiHoster(Hook): def coreReady(self): pluginMap = {} - for name in self.core.pluginManager.hosterPlugins.keys(): + for name in self.core.pluginManager.getPlugins("hoster").keys(): pluginMap[name.lower()] = name new_supported = [] @@ -66,25 +67,19 @@ class MultiHoster(Hook): # inject plugin plugin self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - dict["new_module"] = module - dict["new_name"] = self.__name__ + self.core.pluginManager.injectPlugin("hoster", hoster, module, self.__name__) self.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) # create new regexp regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict["pattern"] = regexp - dict["re"] = re.compile(regexp) + hoster = self.core.pluginManager.getPlugins("hoster") + p = hoster[self.__name__] + new = PluginTuple(p.version, re.compile(regexp), p.deps, p.user, p.path) + hoster[self.__name__] = new def unload(self): for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - if "module" in dict: - del dict["module"] - - del dict["new_module"] - del dict["new_name"] \ No newline at end of file + self.core.pluginManager.restoreState("hoster", hoster) \ No newline at end of file -- cgit v1.2.3 From f852c362fc6283246a9f9e690c456dd3fd245c29 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Fri, 23 Dec 2011 12:15:56 +0100 Subject: closed #473 --- module/plugins/PluginManager.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index 5c31930a5..18dea7699 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -309,18 +309,13 @@ class PluginManager: module = self.loadModule(type, name) if module: return getattr(module, name) - def hasAccountPlugin(self, plugin): - return plugin in self.plugins["accounts"] - - def getAccountPlugins(self): - """return list of account plugin names""" - return self.plugins["accounts"].keys() - def injectPlugin(self, type, name, module, new_name): + """ Overwrite a plugin with a other module. used by Multihoster """ self.modules[(type, name)] = module self.names[(type, name)] = new_name def restoreState(self, type, name): + """ Restore the state of a plugin after injecting """ if (type, name) in self.modules: del self.modules[(type, name)] if (type, name) in self.names: -- cgit v1.2.3 From 9da54933b2bcf58fb0d2342942815ce9e333a400 Mon Sep 17 00:00:00 2001 From: Jeix Date: Sat, 24 Dec 2011 11:25:04 +0100 Subject: ShareOnline Premium, OronCom updates --- module/plugins/accounts/ShareonlineBiz.py | 5 +++-- module/plugins/hoster/OronCom.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/accounts/ShareonlineBiz.py b/module/plugins/accounts/ShareonlineBiz.py index 967142204..4ecd7e1c2 100644 --- a/module/plugins/accounts/ShareonlineBiz.py +++ b/module/plugins/accounts/ShareonlineBiz.py @@ -23,14 +23,15 @@ import re class ShareonlineBiz(Account): __name__ = "ShareonlineBiz" - __version__ = "0.2" + __version__ = "0.3" __type__ = "account" __description__ = """share-online.biz account plugin""" __author_name__ = ("mkaay") __author_mail__ = ("mkaay@mkaay.de") def getUserAPI(self, user, req): - src = req.load("http://api.share-online.biz/account.php?username=%s&password=%s&act=userDetails" % (user, self.accounts[user]["password"])) + data = self.getAccountData(user) + src = req.load("http://api.share-online.biz/account.php?username=%s&password=%s&act=userDetails" % (user, data["password"])) info = {} for line in src.splitlines(): key, value = line.split("=") diff --git a/module/plugins/hoster/OronCom.py b/module/plugins/hoster/OronCom.py index e3e157a6d..0be840d76 100755 --- a/module/plugins/hoster/OronCom.py +++ b/module/plugins/hoster/OronCom.py @@ -33,12 +33,12 @@ class OronCom(Hoster): __name__ = "OronCom" __type__ = "hoster" __pattern__ = r"http://(?:www.)?oron.com/" - __version__ = "0.12" + __version__ = "0.13" __description__ = "File Hoster: Oron.com" __author_name__ = ("chrox", "DHMH") __author_mail__ = ("chrox@pyload.org", "DHMH@pyload.org") - FILE_INFO_PATTERN = r'Dateiname: (.*)
Größe: ([0-9,.]+) (Kb|Mb|Gb)' + FILE_INFO_PATTERN = r'(?:Filename|Dateiname): (.*?)\s*
\s*(?:Größe|File size): ([0-9,\.]+) (Kb|Mb|Gb)' def init(self): self.resumeDownload = self.multiDL = True if self.account else False -- cgit v1.2.3 From 9538fd00584d24c871a3c6b4f47446e187348ff9 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 24 Dec 2011 12:20:41 +0100 Subject: fix SO for new account manager --- module/plugins/Account.py | 21 +++++++++++++++++++-- module/plugins/Plugin.py | 3 ++- module/plugins/accounts/ShareonlineBiz.py | 7 +++---- module/plugins/hoster/ShareonlineBiz.py | 6 +++--- 4 files changed, 27 insertions(+), 10 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Account.py b/module/plugins/Account.py index 9da8d0357..363af3d8b 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -72,15 +72,20 @@ class Account(Base, AccountInfo): def init(self): pass + #TODO: remove user, data def login(self, user, data, req): """login into account, the cookies will be saved so user can be recognized - :param user: loginname - :param data: data dictionary + :param user: Deprecated + :param data: Deprecated :param req: `Request` instance """ raise NotImplemented + def relogin(self): + """ Force a login, same as `_login` """ + return self._login() + @lock def _login(self): # set timestamp for login @@ -106,6 +111,8 @@ class Account(Base, AccountInfo): finally: req.close() + return self.valid + def restoreDefaults(self): self.valid = Account.valid self.validuntil = Account.validuntil @@ -173,6 +180,16 @@ class Account(Base, AccountInfo): else: self.logDebug("Unknown attribute %s=%s" % (k, v)) + #TODO: remove user + def loadAccountInfo(self, user, req): + """ Overwrite this method and set account attributes within this method. + + :param user: Deprecated + :param req: Request instance + :return: + """ + pass + def isPremium(self, user=None): if user: self.logDebug("Deprecated Argument user for .isPremium()", user) return self.premium diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index d78eb162b..3cfb89c21 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -134,7 +134,8 @@ class Plugin(Base): self.thread = thread if self.account: - self.account.checkLogin() + # will force a relogin or reload of account info if necessary + self.account.getAccountInfo() else: self.req.clearCookies() diff --git a/module/plugins/accounts/ShareonlineBiz.py b/module/plugins/accounts/ShareonlineBiz.py index 4ecd7e1c2..4dd398d6d 100644 --- a/module/plugins/accounts/ShareonlineBiz.py +++ b/module/plugins/accounts/ShareonlineBiz.py @@ -29,9 +29,8 @@ class ShareonlineBiz(Account): __author_name__ = ("mkaay") __author_mail__ = ("mkaay@mkaay.de") - def getUserAPI(self, user, req): - data = self.getAccountData(user) - src = req.load("http://api.share-online.biz/account.php?username=%s&password=%s&act=userDetails" % (user, data["password"])) + def getUserAPI(self, req): + src = req.load("http://api.share-online.biz/account.php?username=%s&password=%s&act=userDetails" % (self.loginname, self.password)) info = {} for line in src.splitlines(): key, value = line.split("=") @@ -40,7 +39,7 @@ class ShareonlineBiz(Account): def loadAccountInfo(self, user, req): try: - info = self.getUserAPI(user, req) + info = self.getUserAPI(req) return {"validuntil": int(info["expire_date"]), "trafficleft": -1, "premium": not info["group"] == "Sammler"} except: pass diff --git a/module/plugins/hoster/ShareonlineBiz.py b/module/plugins/hoster/ShareonlineBiz.py index d355eeffe..641a9b025 100644 --- a/module/plugins/hoster/ShareonlineBiz.py +++ b/module/plugins/hoster/ShareonlineBiz.py @@ -52,7 +52,7 @@ class ShareonlineBiz(Hoster): self.multiDL = False self.chunkLimit = 1 - if self.account and self.account.isPremium(self.user): + if self.premium: self.multiDL = True def process(self, pyfile): @@ -60,7 +60,7 @@ class ShareonlineBiz(Hoster): pyfile.name = self.api_data["filename"] pyfile.sync() - if self.account and self.account.isPremium(self.user): + if self.premium: self.handleAPIPremium() #self.handleWebsitePremium() else: @@ -127,7 +127,7 @@ class ShareonlineBiz(Hoster): def handleAPIPremium(self): #should be working better self.resumeDownload = True - info = self.account.getUserAPI(self.user, self.req) + info = self.account.getUserAPI(self.req) if info["dl"].lower() == "not_available": self.fail("DL API error") self.req.cj.setCookie("share-online.biz", "dl", info["dl"]) -- cgit v1.2.3 From 6e64aee6efdbd0ccf7d2d53418fbfaa0765300ef Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 27 Dec 2011 12:33:10 +0100 Subject: some account fixes --- module/plugins/Account.py | 42 +++++++++++++++++++++------------- module/plugins/hoster/RapidshareCom.py | 2 +- 2 files changed, 27 insertions(+), 17 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Account.py b/module/plugins/Account.py index 363af3d8b..86b73c99c 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -18,7 +18,9 @@ class Account(Base, AccountInfo): """ Base class for every Account plugin. Just overwrite `login` and cookies will be stored and account becomes accessible in\ - associated hoster plugin. Plugin should also provide `loadAccountInfo` + associated hoster plugin. Plugin should also provide `loadAccountInfo`. \ + A instance of this class is created for every entered account, it holds all \ + fields of AccountInfo ttype, and can be set easily at runtime. """ # Default values @@ -35,7 +37,7 @@ class Account(Base, AccountInfo): info_threshold = 600 # known options - known_opt = ["time", "limitDL"] + known_opt = ("time", "limitDL") def __init__(self, manager, loginname, password, options): @@ -83,15 +85,18 @@ class Account(Base, AccountInfo): raise NotImplemented def relogin(self): - """ Force a login, same as `_login` """ - return self._login() + """ Force a login. """ + req = self.getAccountRequest() + try: + return self._login(req) + finally: + req.close() @lock - def _login(self): + def _login(self, req): # set timestamp for login self.login_ts = time() - req = self.getAccountRequest() try: self.login(self.loginname, {"password": self.password}, req) self.valid = True @@ -108,8 +113,6 @@ class Account(Base, AccountInfo): self.valid = False if self.core.debug: print_exc() - finally: - req.close() return self.valid @@ -158,16 +161,15 @@ class Account(Base, AccountInfo): if force or self.timestamp + self.info_threshold * 60 < time(): # make sure to login - self.checkLogin() - self.logDebug("Get Account Info for %s" % self.loginname) req = self.getAccountRequest() - + self.checkLogin(req) + self.logDebug("Get Account Info for %s" % self.loginname) try: infos = self.loadAccountInfo(self.loginname, req) except Exception, e: infos = {"error": str(e)} - - req.close() + finally: + req.close() self.logDebug("Account Info: %s" % str(infos)) self.timestamp = time() @@ -181,7 +183,7 @@ class Account(Base, AccountInfo): self.logDebug("Unknown attribute %s=%s" % (k, v)) #TODO: remove user - def loadAccountInfo(self, user, req): + def loadAccountInfo(self, req): """ Overwrite this method and set account attributes within this method. :param user: Deprecated @@ -190,6 +192,14 @@ class Account(Base, AccountInfo): """ pass + def getAccountCookies(self, user): + self.logDebug("Deprecated method .getAccountCookies -> use account.cj") + return self.cj + + def getAccountData(self, user): + self.logDebug("Deprecated method .getAccountData -> use fields directly") + return {"password": self.password} + def isPremium(self, user=None): if user: self.logDebug("Deprecated Argument user for .isPremium()", user) return self.premium @@ -242,7 +252,7 @@ class Account(Base, AccountInfo): self.core.scheduler.addJob(time, self.getAccountInfo, [force]) @lock - def checkLogin(self): + def checkLogin(self, req): """ checks if user is still logged in """ if self.login_ts + self.login_timeout * 60 < time(): if self.login_ts: # seperate from fresh login to have better debug logs @@ -250,7 +260,7 @@ class Account(Base, AccountInfo): else: self.logDebug("Login with %s" % self.loginname) - self._login() + self._login(req) return False return True diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py index f3011a488..a4a72eb53 100644 --- a/module/plugins/hoster/RapidshareCom.py +++ b/module/plugins/hoster/RapidshareCom.py @@ -132,7 +132,7 @@ class RapidshareCom(Hoster): self.handleFree() def handlePremium(self): - info = self.account.getAccountInfo(self.user, True) + info = self.account.getAccountInfo(True) self.log.debug("%s: Use Premium Account" % self.__name__) url = self.api_data["mirror"] self.download(url, get={"directstart":1}) -- cgit v1.2.3 From d35c003cc53d4723d1dfe0d81eeb9bea78cee594 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 31 Dec 2011 16:01:24 +0100 Subject: new crypter plugin API, now decrypting possible for now. --- module/plugins/Account.py | 7 + module/plugins/AccountManager.py | 10 +- module/plugins/Base.py | 101 +++++++- module/plugins/Container.py | 75 ------ module/plugins/Crypter.py | 250 ++++++++++++++----- module/plugins/Hook.py | 2 +- module/plugins/Hoster.py | 444 +++++++++++++++++++++++++++++++++- module/plugins/PluginManager.py | 52 ++-- module/plugins/container/CCF.py | 4 +- module/plugins/container/LinkList.py | 4 +- module/plugins/container/RSDF.py | 4 +- module/plugins/hooks/UpdateManager.py | 5 + 12 files changed, 764 insertions(+), 194 deletions(-) delete mode 100644 module/plugins/Container.py (limited to 'module/plugins') diff --git a/module/plugins/Account.py b/module/plugins/Account.py index 86b73c99c..6b65051db 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -149,6 +149,13 @@ class Account(Base, AccountInfo): def getAccountRequest(self): return self.core.requestFactory.getRequest(self.__name__, self.cj) + def getDownloadSettings(self): + """ Can be overwritten to change download settings. Default is no chunkLimit, multiDL, resumeDownload + + :return: (chunkLimit, multiDL, resumeDownload) / (int,bool,bool) + """ + return -1, True, True + @lock def getAccountInfo(self, force=False): """retrieve account infos for an user, do **not** overwrite this method!\\ diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index c718510ed..77139206c 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -21,7 +21,6 @@ from threading import Lock from random import choice from module.common.json_layer import json -from module.interaction.PullEvents import AccountUpdateEvent from module.utils import lock class AccountManager(): @@ -85,12 +84,15 @@ class AccountManager(): self.createAccount(plugin, user, password, options) self.saveAccounts() + self.sendChange() + @lock def removeAccount(self, plugin, user): """remove account""" if plugin in self.accounts and user in self.accounts[plugin]: del self.accounts[plugin][user] self.core.db.removeAccount(plugin, user) + self.sendChange() else: self.core.log.debug("Remove non existing account %s %s" % (plugin, user)) @@ -118,9 +120,6 @@ class AccountManager(): for acc in p_dict.itervalues(): acc.getAccountInfo() - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) - return self.accounts def refreshAllAccounts(self): @@ -131,5 +130,4 @@ class AccountManager(): def sendChange(self): - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) + self.core.eventManager.dispatchEvent("accountsUpdated") \ No newline at end of file diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 36df7e423..b2338a01f 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -18,12 +18,19 @@ """ import sys +from module.utils.fs import exists, makedirs, join -# TODO: config format definition +# TODO # more attributes if needed # get rid of catpcha & container plugins ?! (move to crypter & internals) # adapt old plugins as needed +class Fail(Exception): + """ raised when failed """ + +class Retry(Exception): + """ raised when start again from beginning """ + class Base(object): """ The Base plugin class with all shared methods and every possible attribute for plugin definition. @@ -31,7 +38,8 @@ class Base(object): __version__ = "0.1" #: Regexp pattern which will be matched for download plugins __pattern__ = r"" - #: Flat config definition + #: Config definition: list of (name, type, verbose_name, default_value) or + #: (name, type, verbose_name, short_description, default_value) __config__ = tuple() #: Short description, one liner __description__ = "" @@ -41,7 +49,7 @@ class Base(object): __dependencies__ = tuple() #: Tags to categorize the plugin __tags__ = tuple() - #: Base64 encoded .png icon + #: Base64 encoded .png icon, please don't use sizes above ~3KB __icon__ = "" #: Alternative, link to png icon __icon_url__ = "" @@ -62,18 +70,25 @@ class Base(object): self.config = core.config #log functions - def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + def logInfo(self, *args, **kwargs): + self._log("info", *args, **kwargs) + + def logWarning(self, *args, **kwargs): + self._log("warning", *args, **kwargs) - def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + def logError(self, *args, **kwargs): + self._log("error", *args, **kwargs) - def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + def logDebug(self, *args, **kwargs): + self._log("debug", *args, **kwargs) - def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + def _log(self, level, *args, **kwargs): + if "sep" in kwargs: + sep = "%s" % kwargs["sep"] + else: + sep = " | " + getattr(self.log, level)("%s: %s" % (self.__name__, sep.join([a if isinstance(a, basestring) else str(a) for a in args]))) def setConf(self, option, value): """ see `setConfig` """ @@ -129,3 +144,67 @@ class Base(object): #noinspection PyUnresolvedReferences sys.stdout = sys._stdout embed() + + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): + """Load content at url and returns it + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param just_header: if True only the header will be retrieved and returned as dict + :param decode: Wether to decode the output according to http header, should be True in most cases + :return: Loaded content + """ + if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") + + if type(url) == unicode: url = str(url) + + res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) + + if self.core.debug: + from inspect import currentframe + + frame = currentframe() + if not exists(join("tmp", self.__name__)): + makedirs(join("tmp", self.__name__)) + + f = open( + join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + , "wb") + del frame # delete the frame or it wont be cleaned + + try: + tmp = res.encode("utf8") + except: + tmp = res + + f.write(tmp) + f.close() + + if just_header: + #parse header + header = {"code": self.req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: continue + + key, none, value = line.partition(":") + key = key.lower().strip() + value = value.strip() + + if key in header: + if type(header[key]) == list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value + res = header + + return res + + def fail(self, reason): + """ fail and give reason """ + raise Fail(reason) \ No newline at end of file diff --git a/module/plugins/Container.py b/module/plugins/Container.py deleted file mode 100644 index c233d3710..000000000 --- a/module/plugins/Container.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from module.plugins.Crypter import Crypter - -from os.path import join, exists, basename -from os import remove -import re - -class Container(Crypter): - __name__ = "Container" - __version__ = "0.1" - __pattern__ = None - __type__ = "container" - __description__ = """Base container plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - - def preprocessing(self, thread): - """prepare""" - - self.setup() - self.thread = thread - - self.loadToDisk() - - self.decrypt(self.pyfile) - self.deleteTmp() - - self.createPackages() - - - def loadToDisk(self): - """loads container to disk if its stored remotely and overwrite url, - or check existent on several places at disk""" - - if self.pyfile.url.startswith("http"): - self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] - content = self.load(self.pyfile.url) - self.pyfile.url = join(self.config["general"]["download_folder"], self.pyfile.name) - f = open(self.pyfile.url, "wb" ) - f.write(content) - f.close() - - else: - self.pyfile.name = basename(self.pyfile.url) - if not exists(self.pyfile.url): - if exists(join(pypath, self.pyfile.url)): - self.pyfile.url = join(pypath, self.pyfile.url) - else: - self.fail(_("File not exists.")) - - - def deleteTmp(self): - if self.pyfile.name.startswith("tmp_"): - remove(self.pyfile.url) - - diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index d1549fe80..fc54b32d7 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -1,72 +1,214 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from module.plugins.Plugin import Plugin - -class Crypter(Plugin): - __name__ = "Crypter" - __version__ = "0.1" - __pattern__ = None - __type__ = "container" - __description__ = """Base crypter plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def __init__(self, pyfile): - Plugin.__init__(self, pyfile) - - #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) - self.packages = [] +from module.Api import Destination +from module.common.packagetools import parseNames +from module.utils import to_list +from module.utils.fs import exists + +from Base import Base, Retry + +class Package: + """ Container that indicates new package should be created """ + def __init__(self, name, urls=None, dest=Destination.Queue): + self.name = name, + self.urls = urls if urls else [] + self.dest = dest + + def addUrl(self, url): + self.urls.append(url) + +class PyFileMockup: + """ Legacy class needed by old crypter plugins """ + def __init__(self, url): + self.url = url + self.name = url + +class Crypter(Base): + """ + Base class for (de)crypter plugins. Overwrite decrypt* methods. + + How to use decrypt* methods + --------------------------- + + You have to overwrite at least one method of decryptURL, decryptURLs, decryptFile. + + After decrypting and generating urls/packages you have to return the result at the\ + end of your method. Valid return Data is: + + `Package` instance + A **new** package will be created with the name and the urls of the object. + + List of urls and `Package` instances + All urls in the list will be added to the **current** package. For each `Package`\ + instance a new package will be created. + + """ - #: List of urls, pyLoad will generate packagenames + @classmethod + def decrypt(cls, core, url_or_urls): + """Static method to decrypt, something. Can be used by other plugins. + + :param core: pyLoad `Core`, needed in decrypt context + :param url_or_urls: List of urls or urls + :return: List of decrypted urls, all packages info removed + """ + urls = to_list(url_or_urls) + p = cls(core) + try: + result = p.processDecrypt(urls) + finally: + p.clean() + + ret = [] + + for url_or_pack in result: + if isinstance(url_or_pack, Package): #package + ret.extend(url_or_pack.urls) + else: # single url + ret.append(url_or_pack) + + return ret + + def __init__(self, core, pid=-1, password=None): + Base.__init__(self, core) + self.req = core.requestFactory.getRequest(self.__name__) + + # Package id plugin was initilized for, dont use this, its not guaranteed to be set + self.pid = pid + + #: Password supplied by user + self.password = password + + # For old style decrypter, do not use these ! + self.packages = [] self.urls = [] - - self.multiDL = True - self.limitDL = 0 - - - def preprocessing(self, thread): - """prepare""" - self.setup() - self.thread = thread - - self.decrypt(self.pyfile) - - self.createPackages() - - - def decrypt(self, pyfile): + self.pyfile = None + + self.init() + + def init(self): + """More init stuff if needed""" + + def setup(self): + """Called everytime before decrypting. A Crypter plugin will be most likly used for several jobs.""" + + def decryptURL(self, url): + """Decrypt a single url + + :param url: url to decrypt + :return: See `Crypter` Documentation + """ + raise NotImplementedError + + def decryptURLs(self, urls): + """Decrypt a bunch of urls + + :param urls: list of urls + :return: See `Crypter` Documentation + """ raise NotImplementedError + def decryptFile(self, content): + """Decrypt file content + + :param content: content to decrypt as string + :return: See `Crypter Documentation + """ + raise NotImplementedError + + def generatePackages(self, urls): + """Generates `Package` instances and names from urls. Usefull for many different link and no\ + given package name. + + :param urls: list of urls + :return: list of `Package` + """ + return [Package(name, purls) for name, purls in parseNames([(url,url) for url in urls]).iteritems()] + + def processDecrypt(self, urls): + """ Internal method to select decrypting method + + :param urls: List of urls/content + :return: + """ + cls = self.__class__ + + # seperate local and remote files + content, urls = self.getLocalContent(urls) + + if hasattr(cls, "decryptURLs"): + self.setup() + result = to_list(self.decryptURLs(urls)) + elif hasattr(cls, "decryptURL"): + result = [] + for url in urls: + self.setup() + result.extend(to_list(self.decryptURL(url))) + elif hasattr(cls, "decrypt"): + self.logDebug("Deprecated .decrypt() method in Crypter plugin") + result = [] # TODO + else: + self.logError("No Decrypting method was overwritten") + result = [] + + if hasattr(cls, "decryptFile"): + for c in content: + self.setup() + result.extend(to_list(self.decryptFile(c))) + + return result + + def getLocalContent(self, urls): + """Load files from disk + + :param urls: + :return: content, remote urls + """ + content = [] + # do nothing if no decryptFile method + if hasattr(self.__class__, "decryptFile"): + remote = [] + for url in urls: + path = None + if url.startswith("http"): + path = None # skip directly + elif exists(url): + path = url + elif exists(self.core.path(url)): + path = self.core.path(url) + + if path: + f = open(path, "wb") + content.append(f.read()) + f.close() + else: + remote.append(url) + + #swap filtered url list + urls = remote + + return content, urls + + def retry(self): + """ Retry decrypting, will only work once. Somewhat deprecated method, should be avoided. """ + raise Retry() + def createPackages(self): - """ create new packages from self.packages """ + """ Deprecated """ + self.logDebug("Deprecated method .createPackages()") for pack in self.packages: self.log.debug("Parsed package %(name)s with %(len)d links" % { "name" : pack[0], "len" : len(pack[1]) } ) links = [x.decode("utf-8") for x in pack[1]] - pid = self.core.api.addPackage(pack[0], links, self.pyfile.package().queue) + pid = self.core.api.files.addLinks(self.pid, links) - if self.pyfile.package().password: - self.core.api.setPackageData(pid, {"password": self.pyfile.package().password}) if self.urls: self.core.api.generateAndAddPackages(self.urls) + def clean(self): + if hasattr(self, "req"): + self.req.close() + del self.req \ No newline at end of file diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index 860dc76bb..a3b86a794 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -20,7 +20,7 @@ from traceback import print_exc -from Plugin import Base +from Base import Base class Expose(object): """ used for decoration to declare rpc services """ diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index aa50099fb..54c2efdfd 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -13,13 +13,39 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - - @author: mkaay + + @author: RaNaN, spoob, mkaay """ -from module.plugins.Plugin import Plugin +from time import time, sleep +from random import randint + +import os + +if os.name != "nt": + from module.utils.fs import chown + from pwd import getpwnam + from grp import getgrnam + +from Base import Base, Fail, Retry +from module.utils import chunks #legacy import +from module.utils.fs import save_join, save_path, fs_encode, fs_decode,\ + remove, makedirs, chmod, stat, exists, join + + +class Abort(Exception): + """ raised when aborted """ + +class Reconnect(Exception): + """ raised when reconnected """ -class Hoster(Plugin): +class SkipDownload(Exception): + """ raised when download should be skipped """ + +class Hoster(Base): + """ + Base plugin for hoster plugin. Overwrite getInfo for online status retrieval, process for downloading. + """ @staticmethod def getInfo(urls): @@ -28,6 +54,412 @@ class Hoster(Plugin): where status is one of API pyfile statusses. :param urls: List of urls - :return: + :return: yield list of tuple with results (name, size, status, url) + """ + pass + + def __init__(self, pyfile): + Base.__init__(self, pyfile.m.core) + + self.wantReconnect = False + #: enables simultaneous processing of multiple downloads + self.multiDL = True + self.limitDL = 0 + #: chunk limit + self.chunkLimit = 1 + #: enables resume (will be ignored if server dont accept chunks) + self.resumeDownload = False + + #: time() + wait in seconds + self.waitUntil = 0 + self.waiting = False + + self.ocr = None #captcha reader instance + #: account handler instance, see :py:class:`Account` + self.account = self.core.accountManager.getAccountForPlugin(self.__name__) + + #: premium status + self.premium = False + #: username/login + self.user = None + + if self.account and not self.account.isUsable(): self.account = None + if self.account: + self.user = self.account.loginname + #: Browser instance, see `network.Browser` + self.req = self.account.getAccountRequest() + # Default: -1, True, True + self.chunkLimit, self.resumeDownload, self.multiDL = self.account.getDownloadSettings() + self.premium = self.account.isPremium() + else: + self.req = self.core.requestFactory.getRequest(self.__name__) + + #: associated pyfile instance, see `PyFile` + self.pyfile = pyfile + self.thread = None # holds thread in future + + #: location where the last call to download was saved + self.lastDownload = "" + #: re match of the last call to `checkDownload` + self.lastCheck = None + #: js engine, see `JsEngine` + self.js = self.core.js + self.cTask = None #captcha task + + self.retries = 0 # amount of retries already made + self.html = None # some plugins store html code here + + self.init() + + def getChunkCount(self): + if self.chunkLimit <= 0: + return self.config["download"]["chunks"] + return min(self.config["download"]["chunks"], self.chunkLimit) + + def __call__(self): + return self.__name__ + + def init(self): + """initialize the plugin (in addition to `__init__`)""" + pass + + def setup(self): + """ setup for enviroment and other things, called before downloading (possibly more than one time)""" + pass + + def preprocessing(self, thread): + """ handles important things to do before starting """ + self.thread = thread + + if self.account: + # will force a relogin or reload of account info if necessary + self.account.getAccountInfo() + else: + self.req.clearCookies() + + self.setup() + + self.pyfile.setStatus("starting") + + return self.process(self.pyfile) + + + def process(self, pyfile): + """the 'main' method of every plugin, you **have to** overwrite it""" + raise NotImplementedError + + def resetAccount(self): + """ dont use account and retry download """ + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retry() + + def checksum(self, local_file=None): + """ + return codes: + 0 - checksum ok + 1 - checksum wrong + 5 - can't get checksum + 10 - not implemented + 20 - unknown error + """ + #@TODO checksum check hook + + return True, 10 + + + def setWait(self, seconds, reconnect=False): + """Set a specific wait time later used with `wait` + + :param seconds: wait time in seconds + :param reconnect: True if a reconnect would avoid wait time + """ + if reconnect: + self.wantReconnect = True + self.pyfile.waitUntil = time() + int(seconds) + + def wait(self): + """ waits the time previously set """ + self.waiting = True + self.pyfile.setStatus("waiting") + + while self.pyfile.waitUntil > time(): + self.thread.m.reconnecting.wait(2) + + if self.pyfile.abort: raise Abort + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + self.waiting = False + self.pyfile.setStatus("starting") + + def offline(self): + """ fail and indicate file is offline """ + raise Fail("offline") + + def tempOffline(self): + """ fail and indicates file ist temporary offline, the core may take consequences """ + raise Fail("temp. offline") + + def retry(self, max_tries=3, wait_time=1, reason=""): + """Retries and begin again from the beginning + + :param max_tries: number of maximum retries + :param wait_time: time to wait in seconds + :param reason: reason for retrying, will be passed to fail if max_tries reached + """ + if 0 < max_tries <= self.retries: + if not reason: reason = "Max retries reached" + raise Fail(reason) + + self.wantReconnect = False + self.setWait(wait_time) + self.wait() + + self.retries += 1 + raise Retry(reason) + + def invalidCaptcha(self): + if self.cTask: + self.cTask.invalid() + + def correctCaptcha(self): + if self.cTask: + self.cTask.correct() + + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', + result_type='textual'): + """ Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param forceUser: if True, ocr is not used + :param imgtype: Type of the Image + :param result_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + + :return: result of decrypting + """ + + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time())[-6:].replace(".", "") + temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + temp_file.write(img) + temp_file.close() + + has_plugin = self.__name__ in self.core.pluginManager.getPlugins("captcha") + + if self.core.captcha: + Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) + else: + Ocr = None + + if Ocr and not forceUser: + sleep(randint(3000, 5000) / 1000.0) + if self.pyfile.abort: raise Abort + + ocr = Ocr() + result = ocr.get_captcha(temp_file.name) + else: + captchaManager = self.core.captchaManager + task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) + self.cTask = task + captchaManager.handleCaptcha(task) + + while task.isWaiting(): + if self.pyfile.abort: + captchaManager.removeTask(task) + raise Abort + sleep(1) + + captchaManager.removeTask(task) + + if task.error and has_plugin: #ignore default error message since the user could use OCR + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) + + result = task.result + self.log.debug("Received captcha result: %s" % str(result)) + + if not self.core.debug: + try: + remove(temp_file.name) + except: + pass + + return result + + + def load(self, *args, **kwargs): + """ See 'Base' load method for more info """ + if self.pyfile.abort: raise Abort + return Base.load(self, *args, **kwargs) + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + """Downloads the content at url to download folder + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param disposition: if True and server provides content-disposition header\ + the filename will be changed if needed + :return: The location where the file was saved + """ + + self.checkForSameFiles() + + self.pyfile.setStatus("downloading") + + download_folder = self.config['general']['download_folder'] + + location = save_join(download_folder, self.pyfile.package().folder) + + if not exists(location): + makedirs(location, int(self.core.config["permission"]["folder"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(location, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + # convert back to unicode + location = fs_decode(location) + name = save_path(self.pyfile.name) + + filename = join(location, name) + + self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) + + try: + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, + chunks=self.getChunkCount(), resume=self.resumeDownload, + progressNotify=self.pyfile.setProgress, disposition=disposition) + finally: + self.pyfile.size = self.req.size + + if disposition and newname and newname != name: #triple check, just to be sure + self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = join(location, newname) + + fs_filename = fs_encode(filename) + + if self.core.config["permission"]["change_file"]: + chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(fs_filename, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + self.lastDownload = filename + return self.lastDownload + + def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): + """ checks the content of the last downloaded file, re match is saved to `lastCheck` + + :param rules: dict with names and rules to match (compiled regexp or strings) + :param api_size: expected file size + :param max_size: if the file is larger then it wont be checked + :param delete: delete if matched + :param read_size: amount of bytes to read from files larger then max_size + :return: dictionary key of the first rule that matched + """ + lastDownload = fs_encode(self.lastDownload) + if not exists(lastDownload): return None + + size = stat(lastDownload) + size = size.st_size + + if api_size and api_size <= size: return None + elif size > max_size and not read_size: return None + self.log.debug("Download Check triggered") + f = open(lastDownload, "rb") + content = f.read(read_size if read_size else -1) + f.close() + #produces encoding errors, better log to other file in the future? + #self.log.debug("Content: %s" % content) + for name, rule in rules.iteritems(): + if type(rule) in (str, unicode): + if rule in content: + if delete: + remove(lastDownload) + return name + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + if delete: + remove(lastDownload) + self.lastCheck = m + return name + + + def getPassword(self): + """ get the password the user provided in the package""" + password = self.pyfile.package().password + if not password: return "" + return password + + + def checkForSameFiles(self, starting=False): + """ checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises SkipDownload: """ - pass \ No newline at end of file + + pack = self.pyfile.package() + + for pyfile in self.core.files.cache.values(): + if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile.status in (0, 12): #finished or downloading + raise SkipDownload(pyfile.pluginname) + elif pyfile.status in ( + 5, 7) and starting: #a download is waiting/starting and was appenrently started before + raise SkipDownload(pyfile.pluginname) + + download_folder = self.config['general']['download_folder'] + location = save_join(download_folder, pack.folder, self.pyfile.name) + + if starting and self.core.config['download']['skip_existing'] and exists(location): + size = os.stat(location).st_size + if size >= self.pyfile.size: + raise SkipDownload("File exists.") + + pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + if pyfile: + if exists(location): + raise SkipDownload(pyfile[0]) + + self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) + + def clean(self): + """ clean everything and remove references """ + if hasattr(self, "pyfile"): + del self.pyfile + if hasattr(self, "req"): + self.req.close() + del self.req + if hasattr(self, "thread"): + del self.thread + if hasattr(self, "html"): + del self.html diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index 18dea7699..e00c1e1f5 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -42,7 +42,7 @@ PluginTuple = namedtuple("PluginTuple", "version re deps user path") class PluginManager: ROOT = "module.plugins." USERROOT = "userplugins." - TYPES = ("crypter", "container", "hoster", "captcha", "accounts", "hooks", "internal") + TYPES = ("crypter", "hoster", "captcha", "accounts", "hooks", "internal") SINGLE = re.compile(r'__(?P[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(? Date: Sun, 1 Jan 2012 13:36:59 +0100 Subject: Happy new Year ! --- module/plugins/Account.py | 30 ++++++------ module/plugins/Crypter.py | 92 +++++++++++++++++++++++------------- module/plugins/container/LinkList.py | 57 ---------------------- module/plugins/crypter/LinkList.py | 51 ++++++++++++++++++++ module/plugins/hoster/YoutubeCom.py | 6 +-- 5 files changed, 128 insertions(+), 108 deletions(-) delete mode 100644 module/plugins/container/LinkList.py create mode 100644 module/plugins/crypter/LinkList.py (limited to 'module/plugins') diff --git a/module/plugins/Account.py b/module/plugins/Account.py index 6b65051db..dcf36f8a0 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -48,10 +48,6 @@ class Account(Base, AccountInfo): else: activated = Account.activated - for opt in self.known_opt: - if opt not in options: - options[opt] = "" - for opt in options.keys(): if opt not in self.known_opt: del options[opt] @@ -74,12 +70,9 @@ class Account(Base, AccountInfo): def init(self): pass - #TODO: remove user, data - def login(self, user, data, req): + def login(self, req): """login into account, the cookies will be saved so user can be recognized - :param user: Deprecated - :param data: Deprecated :param req: `Request` instance """ raise NotImplemented @@ -98,7 +91,13 @@ class Account(Base, AccountInfo): self.login_ts = time() try: - self.login(self.loginname, {"password": self.password}, req) + try: + self.login(req) + except TypeError: #TODO: temporary + self.logDebug("Deprecated .login(...) signature ommit user, data") + self.login(self.loginname, {"password": self.password}, req) + + self.valid = True except WrongPassword: self.logWarning( @@ -117,24 +116,23 @@ class Account(Base, AccountInfo): return self.valid def restoreDefaults(self): - self.valid = Account.valid self.validuntil = Account.validuntil self.trafficleft = Account.trafficleft self.maxtraffic = Account.maxtraffic self.premium = Account.premium - self.activated = Account.activated - def update(self, password=None, options={}): + def update(self, password=None, options=None): """ updates account and return true if anything changed """ self.login_ts = 0 + self.valid = True #set valid so it will be retried to login if "activated" in options: self.activated = from_string(options["avtivated"], "bool") if password: self.password = password - self._login() + self.relogin() return True if options: # remove unknown options @@ -172,7 +170,11 @@ class Account(Base, AccountInfo): self.checkLogin(req) self.logDebug("Get Account Info for %s" % self.loginname) try: - infos = self.loadAccountInfo(self.loginname, req) + try: + infos = self.loadAccountInfo(req) + except TypeError: #TODO: temporary + self.logDebug("Deprecated .loadAccountInfo(...) signature, ommit user argument.") + infos = self.loadAccountInfo(self.loginname, req) except Exception, e: infos = {"error": str(e)} finally: diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index fc54b32d7..7c76afee7 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- +from traceback import print_exc + from module.Api import Destination from module.common.packagetools import parseNames -from module.utils import to_list -from module.utils.fs import exists +from module.utils import to_list, has_method +from module.utils.fs import exists, remove, fs_encode from Base import Base, Retry @@ -17,6 +19,12 @@ class Package: def addUrl(self, url): self.urls.append(url) + def __eq__(self, other): + return self.name == other.name and self.urls == other.urls + + def __repr__(self): + return " Date: Sun, 1 Jan 2012 18:01:25 +0100 Subject: fixed imports --- module/plugins/Crypter.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index 7c76afee7..c645f2a72 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -159,8 +159,8 @@ class Crypter(Base): self.decrypt() result = self.convertPackages() else: - if not has_method(cls, "decryptFile"): - self.logDebug("No Decrypting method was overwritten in plugin %s" % self.__name__) + if not has_method(cls, "decryptFile") or urls: + self.logDebug("No suited decrypting method was overwritten in plugin") result = [] if has_method(cls, "decryptFile"): @@ -199,7 +199,7 @@ class Crypter(Base): for url in urls: path = None if url.startswith("http"): - path = None # skip directly + pass elif exists(url): path = url elif exists(self.core.path(url)): @@ -212,7 +212,6 @@ class Crypter(Base): f.close() except IOError, e: self.logError("IOError", e) - remote.append(url) else: remote.append(url) -- cgit v1.2.3 From 952395d3521ce94a41543d4bdc15f3b73a507397 Mon Sep 17 00:00:00 2001 From: zoidberg10 Date: Tue, 3 Jan 2012 19:39:07 +0100 Subject: update mediafire, easybytez --- module/plugins/accounts/EasybytezCom.py | 68 ++++++++++++++++++++++++++++++++ module/plugins/hoster/EasybytezCom.py | 69 +++++++++++++++++++++++++++++---- module/plugins/hoster/MediafireCom.py | 6 +-- 3 files changed, 133 insertions(+), 10 deletions(-) create mode 100644 module/plugins/accounts/EasybytezCom.py (limited to 'module/plugins') diff --git a/module/plugins/accounts/EasybytezCom.py b/module/plugins/accounts/EasybytezCom.py new file mode 100644 index 000000000..cf2b16394 --- /dev/null +++ b/module/plugins/accounts/EasybytezCom.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: zoidberg +""" + +from module.plugins.Account import Account +import re +from module.utils import parseFileSize +from time import mktime, strptime + +class EasybytezCom(Account): + __name__ = "EasybytezCom" + __version__ = "0.01" + __type__ = "account" + __description__ = """EasyBytez.com account plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + VALID_UNTIL_PATTERN = r'Premium account expire:([^<]+)' + TRAFFIC_LEFT_PATTERN = r'Traffic available today:(?P[^<]+)' + + def loadAccountInfo(self, user, req): + #self.relogin(user) + html = req.load("http://www.easybytez.com/?op=my_account", decode = True) + + validuntil = -1 + found = re.search(self.VALID_UNTIL_PATTERN, html) + if found: + premium = True + try: + self.logDebug(found.group(1)) + validuntil = mktime(strptime(found.group(1), "%d %B %Y")) + except Exception, e: + self.logError(e) + else: + premium = False + + #found = re.search(self.TRAFFIC_LEFT_PATTERN, html) + #trafficleft = parseFileSize(found.group('S')) / 1024 if found else 0 + #self.premium = True if trafficleft else False + trafficleft = -1 + + return ({"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium}) + + def login(self, user, data, req): + html = req.load('http://www.easybytez.com/', post = { + "login": user, + "op": "login", + "password": data['password'], + "redirect": "http://easybytez.com/" + }, decode = True) + + if 'Incorrect Login or Password' in html: + self.wrongPassword() \ No newline at end of file diff --git a/module/plugins/hoster/EasybytezCom.py b/module/plugins/hoster/EasybytezCom.py index 5858dd935..dac35b1d3 100644 --- a/module/plugins/hoster/EasybytezCom.py +++ b/module/plugins/hoster/EasybytezCom.py @@ -18,12 +18,13 @@ import re from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from random import random class EasybytezCom(SimpleHoster): __name__ = "EasybytezCom" __type__ = "hoster" __pattern__ = r"http://(?:\w*\.)?easybytez.com/(\w+).*" - __version__ = "0.01" + __version__ = "0.03" __description__ = """easybytez.com""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") @@ -36,24 +37,78 @@ class EasybytezCom(SimpleHoster): FORM_INPUT_PATTERN = r']* name="([^"]+)" value="([^"]*)">' WAIT_PATTERN = r'[^>]*>(\d+) seconds' + DIRECT_LINK_PATTERN = r'(http://\w+\.easybytez\.com/files/\d+/\w+/[^"<]+)' + URL_FORM_PATTERN = r'
]*action="([^"]+)(.*?)
' + OVR_DOWNLOAD_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' + OVR_KILL_LINK_PATTERN = r'

Delete Link

\s*]*>([^<]+)' + + def process(self, pyfile): + if not re.match(self.__pattern__, self.pyfile.url): + if self.premium: + self.handleOverriden() + else: + self.fail("Only premium users can download from other hosters with EasyBytes") + else: + self.html = self.load(pyfile.url, cookies = False, decode = True) + self.file_info = self.getFileInfo() + + header = self.load(self.pyfile.url, just_header = True, cookies = True) + self.logDebug(header) + + if 'location' in header and re.match(self.DIRECT_LINK_PATTERN, header['location']): + self.downloadLink(header['location']) + elif self.premium: + self.handlePremium() + else: + self.handleFree() + def handleFree(self): self.download(self.pyfile.url, post = self.getPostParameters(), ref = True, cookies = True) + + def handlePremium(self): + self.html = self.load(self.pyfile.url, post = self.getPostParameters(premium=True)) + found = re.search(self.DIRECT_LINK_PATTERN, self.html) + if not found: self.parseError('DIRECT LINK') + self.downloadLink(found.group(1)) + + def handleOverriden(self): + self.html = self.load('http://www.easybytez.com/') + action, form = re.search(self.URL_FORM_PATTERN, self.html, re.DOTALL).groups() + inputs = dict(re.findall(self.FORM_INPUT_PATTERN, form)) + action += "%d&js_on=1&utype=prem&upload_type=url" % int(random()*10**12) + inputs['tos'] = '1' + inputs['url_mass'] = self.pyfile.url + + self.html = self.load(action, post = inputs) + found = re.search(self.OVR_DOWNLOAD_LINK_PATTERN, self.html) + if not found: self.parseError('DIRECT LINK (OVR)') + self.downloadLink(found.group(1)) - def getPostParameters(self): + def downloadLink(self, link): + self.logDebug('DIRECT LINK: %s' % link) + self.download(link) + + def getPostParameters(self, premium=False): inputs = dict(re.findall(self.FORM_INPUT_PATTERN, self.html)) self.logDebug(inputs) - inputs['method_free'] = "Free Download" inputs['referer'] = self.pyfile.url - if 'method_premium' in inputs: del inputs['method_premium'] + + if premium: + inputs['method_premium'] = "Premium Download" + if 'method_free' in inputs: del inputs['method_free'] + else: + inputs['method_free'] = "Free Download" + if 'method_premium' in inputs: del inputs['method_premium'] self.html = self.load(self.pyfile.url, post = inputs, ref = True, cookies = True) inputs = dict(re.findall(self.FORM_INPUT_PATTERN, self.html)) self.logDebug(inputs) - found = re.search(self.WAIT_PATTERN, self.html) - self.setWait(int(found.group(1)) + 1 if found else 60) - self.wait() + if not premium: + found = re.search(self.WAIT_PATTERN, self.html) + self.setWait(int(found.group(1)) + 1 if found else 60) + self.wait() return inputs diff --git a/module/plugins/hoster/MediafireCom.py b/module/plugins/hoster/MediafireCom.py index 484b48ba6..14180ff3d 100644 --- a/module/plugins/hoster/MediafireCom.py +++ b/module/plugins/hoster/MediafireCom.py @@ -58,20 +58,20 @@ class MediafireCom(SimpleHoster): __name__ = "MediafireCom" __type__ = "hoster" __pattern__ = r"http://(?:\w*\.)*mediafire\.com/[^?].*" - __version__ = "0.70" + __version__ = "0.71" __description__ = """Mediafire.com plugin - free only""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") DOWNLOAD_LINK_PATTERN = r'
]*z-index:(?P\d+)[^>]*>\s*' PAGE1_ACTION_PATTERN = r'' PASSWORD_PATTERN = r";break;}\s*dh\('" FILE_NAME_PATTERN = r'' - FILE_SIZE_PATTERN = r'>Download \((?P[0-9.]+) (?P[kKMG])i?B">\)' + FILE_SIZE_PATTERN = r'>Download\s*\((?P[^)]+)\)' FILE_OFFLINE_PATTERN = r'class="error_msg_title"> Invalid or Deleted File.
' def process(self, pyfile): -- cgit v1.2.3 From 4a3a81b63cd85cc3dcd9669868a2079da65838a2 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 3 Jan 2012 20:41:23 +0100 Subject: fixes for old style decrypter --- module/plugins/Base.py | 17 ++++------- module/plugins/Crypter.py | 61 +++++++++++++++++++++++++------------- module/plugins/crypter/LinkList.py | 4 +++ 3 files changed, 50 insertions(+), 32 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Base.py b/module/plugins/Base.py index b2338a01f..0ad0d5caa 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -90,18 +90,13 @@ class Base(object): getattr(self.log, level)("%s: %s" % (self.__name__, sep.join([a if isinstance(a, basestring) else str(a) for a in args]))) - def setConf(self, option, value): - """ see `setConfig` """ - self.core.config.set(self.__name__, option, value) - def setConfig(self, option, value): """ Set config value for current plugin :param option: :param value: - :return: """ - self.setConf(option, value) + self.core.config.set(self.__name__, option, value) def getConf(self, option): """ see `getConfig` """ @@ -148,11 +143,11 @@ class Base(object): def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """Load content at url and returns it - :param url: - :param get: - :param post: - :param ref: - :param cookies: + :param url: url as string + :param get: GET as dict + :param post: POST as dict + :param ref: Set HTTP_REFERER header + :param cookies: use saved cookies :param just_header: if True only the header will be retrieved and returned as dict :param decode: Wether to decode the output according to http header, should be True in most cases :return: Loaded content diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index c645f2a72..fe7f0deb8 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -27,23 +27,27 @@ class Package: class PyFileMockup: """ Legacy class needed by old crypter plugins """ - def __init__(self, url): + def __init__(self, url, pack): self.url = url self.name = url + self._package = pack + self.packageid = pack.id if pack else -1 + + def package(self): + return self._package class Crypter(Base): """ Base class for (de)crypter plugins. Overwrite decrypt* methods. - How to use decrypt* methods - --------------------------- + How to use decrypt* methods: You have to overwrite at least one method of decryptURL, decryptURLs, decryptFile. - After decrypting and generating urls/packages you have to return the result at the\ - end of your method. Valid return Data is: + After decrypting and generating urls/packages you have to return the result. + Valid return Data is: - `Package` instance + :class:`Package` instance Crypter.Package A **new** package will be created with the name and the urls of the object. List of urls and `Package` instances @@ -52,9 +56,13 @@ class Crypter(Base): """ + #: Prefix to annotate that the submited string for decrypting is indeed file content + CONTENT_PREFIX = "filecontent:" + @classmethod def decrypt(cls, core, url_or_urls): """Static method to decrypt, something. Can be used by other plugins. + To decrypt file content prefix the string with ``CONTENT_PREFIX `` as seen above. :param core: pyLoad `Core`, needed in decrypt context :param url_or_urls: List of urls or single url/ file content @@ -105,15 +113,18 @@ class Crypter(Base): """Decrypt a single url :param url: url to decrypt - :return: See `Crypter` Documentation + :return: See :class:`Crypter` Documentation """ - raise NotImplementedError + if url.startswith("http"): # basic method to redirect + return self.decryptFile(self.load(url)) + else: + self.fail(_("Not existing file or unsupported protocol")) def decryptURLs(self, urls): """Decrypt a bunch of urls :param urls: list of urls - :return: See `Crypter` Documentation + :return: See :class:`Crypter` Documentation """ raise NotImplementedError @@ -121,12 +132,12 @@ class Crypter(Base): """Decrypt file content :param content: content to decrypt as string - :return: See `Crypter Documentation + :return: See :class:`Crypter` Documentation """ raise NotImplementedError def generatePackages(self, urls): - """Generates `Package` instances and names from urls. Usefull for many different link and no\ + """Generates :class:`Package` instances and names from urls. Usefull for many different links and no\ given package name. :param urls: list of urls @@ -155,9 +166,12 @@ class Crypter(Base): result.extend(to_list(self.decryptURL(url))) elif has_method(cls, "decrypt"): self.logDebug("Deprecated .decrypt() method in Crypter plugin") - self.setup() - self.decrypt() - result = self.convertPackages() + result = [] + for url in urls: + self.pyfile = PyFileMockup(url, self.package) + self.setup() + self.decrypt(self.pyfile) + result.extend(self.convertPackages()) else: if not has_method(cls, "decryptFile") or urls: self.logDebug("No suited decrypting method was overwritten in plugin") @@ -175,7 +189,7 @@ class Crypter(Base): return result def processDecrypt(self, urls): - """ Catches all exceptions in decrypt methods and return results + """Catches all exceptions in decrypt methods and return results :return: Decrypting results """ @@ -187,10 +201,10 @@ class Crypter(Base): return [] def getLocalContent(self, urls): - """Load files from disk + """Load files from disk and seperate to file content and url list :param urls: - :return: content, remote urls + :return: list of (filename, content), remote urls """ content = [] # do nothing if no decryptFile method @@ -198,8 +212,10 @@ class Crypter(Base): remote = [] for url in urls: path = None - if url.startswith("http"): + if url.startswith("http"): # skip urls directly pass + elif url.startswith(self.CONTENT_PREFIX): + path = url elif exists(url): path = url elif exists(self.core.path(url)): @@ -207,9 +223,12 @@ class Crypter(Base): if path: try: - f = open(fs_encode(path), "rb") - content.append((f.name, f.read())) - f.close() + if path.startswith(self.CONTENT_PREFIX): + content.append(("", path[len(self.CONTENT_PREFIX)])) + else: + f = open(fs_encode(path), "rb") + content.append((f.name, f.read())) + f.close() except IOError, e: self.logError("IOError", e) else: diff --git a/module/plugins/crypter/LinkList.py b/module/plugins/crypter/LinkList.py index 8e46f88a9..ebfa373eb 100644 --- a/module/plugins/crypter/LinkList.py +++ b/module/plugins/crypter/LinkList.py @@ -11,6 +11,10 @@ class LinkList(Crypter): __author_name__ = ("spoob", "jeix") __author_mail__ = ("spoob@pyload.org", "jeix@hasnomail.com") + # method declaration is needed here + def decryptURL(self, url): + return Crypter.decryptURL(self, url) + def decryptFile(self, content): links = content.splitlines() -- cgit v1.2.3 From 18466eb7f8f3cd4ca9a0824074d2ff454939fce6 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Wed, 4 Jan 2012 17:23:13 +0100 Subject: some fixes --- module/plugins/Account.py | 4 ++++ module/plugins/AccountManager.py | 2 +- module/plugins/Base.py | 12 ++++++++-- module/plugins/Hook.py | 48 ++++++++++++++++++++++------------------ 4 files changed, 42 insertions(+), 24 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Account.py b/module/plugins/Account.py index dcf36f8a0..59ce87ed2 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -48,6 +48,10 @@ class Account(Base, AccountInfo): else: activated = Account.activated + for opt in self.known_opt: + if opt not in options: + options[opt] = "" + for opt in options.keys(): if opt not in self.known_opt: del options[opt] diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index 77139206c..00dd2ccc6 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -23,7 +23,7 @@ from random import choice from module.common.json_layer import json from module.utils import lock -class AccountManager(): +class AccountManager: """manages all accounts""" def __init__(self, core): diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 0ad0d5caa..1477356ea 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -40,7 +40,7 @@ class Base(object): __pattern__ = r"" #: Config definition: list of (name, type, verbose_name, default_value) or #: (name, type, verbose_name, short_description, default_value) - __config__ = tuple() + __config__ = list() #: Short description, one liner __description__ = "" #: More detailed text @@ -68,9 +68,17 @@ class Base(object): self.log = core.log #: core config self.config = core.config + #: :class:`EventManager` + self.evm = core.eventManager + #: :class:`InteractionManager` + self.im = core.interActionManager - #log functions def logInfo(self, *args, **kwargs): + """ Print args to log at specific level + + :param args: Arbitary object which should be logged + :param kwargs: sep=(how to seperate arguments), default = " | " + """ self._log("info", *args, **kwargs) def logWarning(self, *args, **kwargs): diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index a3b86a794..fe464bdaa 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -14,8 +14,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - @author: mkaay - @interface-version: 0.2 + @author: RaNaN """ from traceback import print_exc @@ -24,11 +23,24 @@ from Base import Base class Expose(object): """ used for decoration to declare rpc services """ - def __new__(cls, f, *args, **kwargs): hookManager.addRPC(f.__module__, f.func_name, f.func_doc) return f +def AddEventListener(event): + """ used to register method for events """ + class _klass(object): + def __new__(cls, f, *args, **kwargs): + hookManager.addEventListener(f.__module__, f.func_name, event) + return f + return _klass + +class ConfigHandler(object): + """ register method as config handler """ + def __new__(cls, f, *args, **kwargs): + hookManager.addConfigHandler(f.__module__, f.func_name) + return f + def threaded(f): def run(*args,**kwargs): hookManager.startThread(f, *args, **kwargs) @@ -38,14 +50,6 @@ class Hook(Base): """ Base class for hook plugins. """ - __name__ = "Hook" - __version__ = "0.2" - __type__ = "hook" - __threaded__ = [] - __config__ = [ ("name", "type", "desc" , "default") ] - __description__ = """interface for hook""" - __author_name__ = ("mkaay", "RaNaN") - __author_mail__ = ("mkaay@mkaay.de", "RaNaN@pyload.org") #: automatically register event listeners for functions, attribute will be deleted dont use it yourself event_map = None @@ -108,7 +112,11 @@ class Hook(Base): def __repr__(self): return "" % self.__name__ - + + def isActivated(self): + """ checks if hook is activated""" + return self.getConfig("activated") + def setup(self): """ more init stuff if needed """ pass @@ -116,11 +124,12 @@ class Hook(Base): def unload(self): """ called when hook was deactivated """ pass - - def isActivated(self): - """ checks if hook is activated""" - return self.config.get(self.__name__, "activated") - + + def deactivate(self): + pass + + def activate(self): + pass #event methods - overwrite these if needed def coreReady(self): @@ -134,10 +143,7 @@ class Hook(Base): def downloadFinished(self, pyfile): pass - - def downloadFailed(self, pyfile): - pass - + def packageFinished(self, pypack): pass -- cgit v1.2.3 From e370517032aeb5f8305d9e0fdc26f643ff30f883 Mon Sep 17 00:00:00 2001 From: zoidberg10 Date: Wed, 4 Jan 2012 17:52:43 +0100 Subject: filesonic pattern fix (thanx Lino24), fix mu life account, closed #463 easybytez --- module/plugins/accounts/MegauploadCom.py | 6 ++--- module/plugins/hooks/EasybytezCom.py | 32 +++++++++++++++++++++++ module/plugins/hoster/CrockoCom.py | 5 ++-- module/plugins/hoster/EasybytezCom.py | 45 ++++++++++++++++++++++++++------ module/plugins/hoster/FilesonicCom.py | 6 ++--- 5 files changed, 78 insertions(+), 16 deletions(-) create mode 100644 module/plugins/hooks/EasybytezCom.py (limited to 'module/plugins') diff --git a/module/plugins/accounts/MegauploadCom.py b/module/plugins/accounts/MegauploadCom.py index 12e510fcf..ff4f5971c 100644 --- a/module/plugins/accounts/MegauploadCom.py +++ b/module/plugins/accounts/MegauploadCom.py @@ -24,16 +24,16 @@ from module.plugins.Account import Account class MegauploadCom(Account): __name__ = "MegauploadCom" - __version__ = "0.11" + __version__ = "0.12" __type__ = "account" __description__ = """megaupload account plugin""" __author_name__ = ("RaNaN") __author_mail__ = ("RaNaN@pyload.org") def loadAccountInfo(self, user, req): - page = req.load("http://www.megaupload.com/?c=account") + page = req.load("http://www.megaupload.com/?c=account&setlang=en", decode = True) - premium = True if r'' LINK_PATTERN = r'' - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) + def decryptURL(self, url): + html = self.load(url) new_links = [] @@ -27,6 +27,7 @@ class OronComFolder(Crypter): new_links.extend(re.findall(self.LINK_PATTERN, folder.group(0))) if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) + self.logDebug("Found %d new links" % len(new_links)) + return new_links else: - self.fail('Could not extract any links') \ No newline at end of file + self.fail('Could not extract any links') -- cgit v1.2.3 From 13c23eb348f9115b2c6b47a579b99d8d1150522b Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 5 Feb 2012 05:16:16 +0000 Subject: Fix OronComFolder regex folder matching. --- module/plugins/crypter/OronComFolder.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/crypter/OronComFolder.py b/module/plugins/crypter/OronComFolder.py index 91ac5435f..726371966 100755 --- a/module/plugins/crypter/OronComFolder.py +++ b/module/plugins/crypter/OronComFolder.py @@ -13,7 +13,7 @@ class OronComFolder(Crypter): __author_name__ = ("DHMH") __author_mail__ = ("webmaster@pcProfil.de") - FOLDER_PATTERN = r'(.*)
\n ' + FOLDER_PATTERN = r'(?:.*)(?P.*)(?:.*)' LINK_PATTERN = r'' def decryptURL(self, url): @@ -21,8 +21,18 @@ class OronComFolder(Crypter): new_links = [] + if 'No such folder exist' in html: + # Don't fail because if there's more than a folder for this package + # and only one of them fails, no urls at all will be added. + self.logWarning("Folder does not exist") + return new_links + folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) - if folder is None: self.fail("Parse error (FOLDER)") + if folder is None: + # Don't fail because if there's more than a folder for this package + # and only one of them fails, no urls at all will be added. + self.logWarning("Parse error (FOLDER)") + return new_links new_links.extend(re.findall(self.LINK_PATTERN, folder.group(0))) @@ -30,4 +40,7 @@ class OronComFolder(Crypter): self.logDebug("Found %d new links" % len(new_links)) return new_links else: - self.fail('Could not extract any links') + # Don't fail because if there's more than a folder for this package + # and only one of them fails, no urls at all will be added. + self.logWarning('Could not extract any links') + return new_links -- cgit v1.2.3 From 69230e264359f5a53faec5ddb2f768255cab0c77 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 5 Feb 2012 05:16:58 +0000 Subject: Make sure that the Oron hoster plugins does not match Oron folders. --- module/plugins/hoster/OronCom.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/hoster/OronCom.py b/module/plugins/hoster/OronCom.py index e0be91486..e659beee5 100755 --- a/module/plugins/hoster/OronCom.py +++ b/module/plugins/hoster/OronCom.py @@ -31,8 +31,8 @@ def getInfo(urls): class OronCom(Hoster): __name__ = "OronCom" __type__ = "hoster" - __pattern__ = r"http://(?:www.)?oron.com/" - __version__ = "0.13" + __pattern__ = r"http://(?:www\.)?oron.com/(?!folder/)" + __version__ = "0.14" __description__ = "File Hoster: Oron.com" __author_name__ = ("chrox", "DHMH") __author_mail__ = ("chrox@pyload.org", "DHMH@pyload.org") -- cgit v1.2.3 From f04e77d96c7483cae90f19eb8468557a44fa49e5 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 5 Feb 2012 06:41:43 +0000 Subject: Stop Real-Debrid from returning streaming responses instead of file downloads. --- module/plugins/hoster/RealdebridCom.py | 35 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 19 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/hoster/RealdebridCom.py b/module/plugins/hoster/RealdebridCom.py index 3a3ac99b0..5759838e2 100644 --- a/module/plugins/hoster/RealdebridCom.py +++ b/module/plugins/hoster/RealdebridCom.py @@ -2,14 +2,16 @@ # -*- coding: utf-8 -*- import re +from time import time from urllib import quote, unquote from random import randrange -from module.utils import encode +from module.utils import encode, parseFileSize +from module.common.json_layer import json_loads from module.plugins.Hoster import Hoster class RealdebridCom(Hoster): - __version__ = "0.41" + __version__ = "0.42" __pattern__ = r"https?://.*real-debrid\..*" __description__ = """Real-Debrid.com hoster plugin""" __config__ = [("https", "bool", _("Enable HTTPS"), False)] @@ -45,28 +47,24 @@ class RealdebridCom(Hoster): if not password: password = "" else: password = password[0] - url = "http://real-debrid.com/ajax/deb.php?lang=en&sl=1&link=%s&passwort=%s" % (quote(encode(pyfile.url), ""), password) + url = "http://real-debrid.com/ajax/unrestrict.php?lang=en&link=%s&password=%s&time=%s" % (quote(encode(pyfile.url), ""), password, int(time()*1000)) page = self.load(url) + data = json_loads(page) - error = re.search(r'(.*)', page) - generation_ok = re.search(r'(.*)', page) - if generation_ok: - page = generation_ok.group(1).strip() + self.logDebug("Returned Data: %s" % data) - if error: - msg = error.group(1).strip() - self.logDebug(page) - if msg == "Your file is unavailable on the hoster.": + if data["error"] != 0: + if data["message"] == "Your file is unavailable on the hoster.": self.offline() + elif data["message"] == "File's hoster is in maintenance. Try again later.": + self.logWarning(data["message"]) + self.tempOffline() else: - self.fail(msg) - elif url == 'error': - self.fail("Your IP is most likely blocked. Please contact RealDebrid support") - elif page == "File's hoster is in maintenance. Try again later.": - self.logWarning(page) - self.tempOffline() + self.logError(page) else: - new_url = page + self.pyfile.name = data["file_name"] + self.pyfile.size = parseFileSize(data["file_size"]) + new_url = data['generated_links'].split('|')[-1] if self.getConfig("https"): new_url = new_url.replace("http://", "https://") @@ -75,7 +73,6 @@ class RealdebridCom(Hoster): self.log.debug("Real-Debrid: New URL: %s" % new_url) - if pyfile.name.startswith("http") or pyfile.name.startswith("Unknown"): #only use when name wasnt already set pyfile.name = self.getFilename(new_url) -- cgit v1.2.3 From e8a0cc2daf8cf5b0ad6d1faee53cd5551fa6dfaa Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 5 Feb 2012 11:00:31 +0000 Subject: Fix missing error message. --- module/plugins/hoster/RealdebridCom.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'module/plugins') diff --git a/module/plugins/hoster/RealdebridCom.py b/module/plugins/hoster/RealdebridCom.py index 5759838e2..d1f87f3dd 100644 --- a/module/plugins/hoster/RealdebridCom.py +++ b/module/plugins/hoster/RealdebridCom.py @@ -11,7 +11,7 @@ from module.common.json_layer import json_loads from module.plugins.Hoster import Hoster class RealdebridCom(Hoster): - __version__ = "0.42" + __version__ = "0.43" __pattern__ = r"https?://.*real-debrid\..*" __description__ = """Real-Debrid.com hoster plugin""" __config__ = [("https", "bool", _("Enable HTTPS"), False)] @@ -59,6 +59,9 @@ class RealdebridCom(Hoster): elif data["message"] == "File's hoster is in maintenance. Try again later.": self.logWarning(data["message"]) self.tempOffline() + elif data["message"] == "No server is available for this hoster.": + self.logWarning(data["message"]) + self.tempOffline() else: self.logError(page) else: -- cgit v1.2.3 From c318d10d2b85160f892eb0ddfdbe295fa0d46aa4 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Sun, 5 Feb 2012 11:04:14 +0000 Subject: Treat most errors as tempfailures, this will solve the missing errors. --- module/plugins/hoster/RealdebridCom.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/hoster/RealdebridCom.py b/module/plugins/hoster/RealdebridCom.py index d1f87f3dd..46ac51c82 100644 --- a/module/plugins/hoster/RealdebridCom.py +++ b/module/plugins/hoster/RealdebridCom.py @@ -56,14 +56,9 @@ class RealdebridCom(Hoster): if data["error"] != 0: if data["message"] == "Your file is unavailable on the hoster.": self.offline() - elif data["message"] == "File's hoster is in maintenance. Try again later.": - self.logWarning(data["message"]) - self.tempOffline() - elif data["message"] == "No server is available for this hoster.": + else: self.logWarning(data["message"]) self.tempOffline() - else: - self.logError(page) else: self.pyfile.name = data["file_name"] self.pyfile.size = parseFileSize(data["file_size"]) -- cgit v1.2.3 From 7bd2ffe0e50efea468efaec28abace2055dab42d Mon Sep 17 00:00:00 2001 From: Jeix Date: Sun, 5 Feb 2012 14:31:50 +0100 Subject: closed #261 --- module/plugins/hoster/Xdcc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/hoster/Xdcc.py b/module/plugins/hoster/Xdcc.py index 7d83b050c..6f0a1b176 100644 --- a/module/plugins/hoster/Xdcc.py +++ b/module/plugins/hoster/Xdcc.py @@ -215,7 +215,7 @@ class Xdcc(Hoster): self.log.info("XDCC: Downloading %s from %s:%d" % (packname, ip, port)) self.pyfile.setStatus("downloading") - newname = self.req.download(ip, port, filename, self.pyfile.setProgress) + newname = self.req.download(ip, port, filename, sock, self.pyfile.setProgress) if newname and newname != filename: self.log.info("%(name)s saved as %(newname)s" % {"name": self.pyfile.name, "newname": newname}) filename = newname @@ -227,4 +227,3 @@ class Xdcc(Hoster): self.lastDownload = filename return self.lastDownload - -- cgit v1.2.3 From d7eef2c28eae2e43e3ade4441810ecc0cdea6fd7 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 5 Feb 2012 21:21:36 +0100 Subject: option for internal plugins --- module/plugins/Base.py | 2 ++ module/plugins/Hook.py | 2 +- module/plugins/PluginManager.py | 53 ++++++++++++++++++++++--------------- module/plugins/hooks/MultiHoster.py | 3 ++- 4 files changed, 36 insertions(+), 24 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 34074095e..29ff3a723 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -39,6 +39,8 @@ class Base(object): __version__ = "0.1" #: Regexp pattern which will be matched for download/crypter plugins __pattern__ = r"" + #: Internal Hook plugin which is always loaded + __internal__ = False #: Config definition: list of (name, type, verbose_name, default_value) or #: (name, type, verbose_name, short_description, default_value) __config__ = list() diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index 83ef091ae..22765c525 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -124,7 +124,7 @@ class Hook(Base): def isActivated(self): """ checks if hook is activated""" - return self.getConfig("activated") + return True if self.__internal__ else self.getConfig("activated") def init(self): pass diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index c345f765f..4e2fa21ed 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -24,7 +24,6 @@ from os import listdir, makedirs from os.path import isfile, join, exists, abspath, basename from sys import version_info from time import time -from traceback import print_exc from module.lib.SafeEval import const_eval as literal_eval from module.plugins.Base import Base @@ -44,9 +43,9 @@ class PluginManager: USERROOT = "userplugins." TYPES = ("crypter", "hoster", "captcha", "accounts", "hooks", "internal") + BUILTIN = re.compile(r'__(?P[a-z0-9_]+)__\s*=\s?(True|False|None|[0-9x.]+)', re.I) SINGLE = re.compile(r'__(?P[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?[a-z0-9_]+)__\s*=\s*((?:\{|\[|"{3}).*?(?:"""|\}|\]))', re.DOTALL | re.M | re.I) @@ -58,10 +57,9 @@ class PluginManager: self.plugins = {} self.modules = {} # cached modules - self.history = [] # match history to speedup parsing (type, name) + self.history = [] # match history to speedup parsing (type, name) self.createIndex() - self.core.config.parseValues(self.core.config.PLUGIN) #register for import hook @@ -126,28 +124,36 @@ class PluginManager: return plugins - def parsePlugin(self, filename, folder, name, home=None): - """ Parses a plugin from disk, folder means plugin type in this context. Also sets config. - - :arg home: dict with plugins, of which the found one will be matched against (according version) - :returns PluginTuple""" - + def parseAttributes(self, filename, name, folder=""): + """ Parse attribute dict from plugin""" data = open(filename, "rb") content = data.read() data.close() attrs = {} - for m in self.SINGLE.findall(content) + self.MULTI.findall(content): + for m in self.BUILTIN.findall(content) + self.SINGLE.findall(content) + self.MULTI.findall(content): #replace gettext function and eval result try: attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) except: self.logDebug(folder, name, "Error when parsing: %s" % m[-1]) - return + self.core.print_exc() + if not hasattr(Base, "__%s__" % m[0]): if m[0] != "type": #TODO remove type from all plugins, its not needed self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) + return attrs + + def parsePlugin(self, filename, folder, name, home=None): + """ Parses a plugin from disk, folder means plugin type in this context. Also sets config. + + :arg home: dict with plugins, of which the found one will be matched against (according version) + :returns PluginTuple""" + + attrs = self.parseAttributes(filename, name, folder) + if not attrs: return + version = 0 if "version" in attrs: @@ -185,7 +191,7 @@ class PluginManager: if folder == "internal": return plugin - if folder == "hooks" and "config" not in attrs: + if folder == "hooks" and "config" not in attrs and not attrs.get("internal", False): attrs["config"] = (["activated", "bool", "Activated", False],) if "config" in attrs and attrs["config"]: @@ -198,13 +204,11 @@ class PluginManager: else: config = [list(config)] - if folder == "hooks": - append = True + if folder == "hooks" and not attrs.get("internal", False): for item in config: - if item[0] == "activated": append = False - - # activated flag missing - if append: config.insert(0, ("activated", "bool", "Activated", False)) + if item[0] == "activated": break + else: # activated flag missing + config.insert(0, ("activated", "bool", "Activated", False)) try: self.core.config.addConfigSection(name, name, desc, long_desc, config) @@ -230,7 +234,7 @@ class PluginManager: if self.plugins[ptype][name].re.match(url): res[ptype].append((url, name)) found = (ptype, name) - break + break # need to exit this loop first if found: # found match if self.history[0] != found: #update history @@ -275,6 +279,12 @@ class PluginManager: # MultiHoster will overwrite this getPlugin = getPluginClass + + def loadAttributes(self, type, name): + plugin = self.plugins[type][name] + return self.parseAttributes(plugin.path, name, type) + + def loadModule(self, type, name): """ Returns loaded module for plugin @@ -292,8 +302,7 @@ class PluginManager: return module except Exception, e: self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) - if self.core.debug: - print_exc() + self.core.print_exc() def loadClass(self, type, name): """Returns the class of a plugin with the same name""" diff --git a/module/plugins/hooks/MultiHoster.py b/module/plugins/hooks/MultiHoster.py index 749f2c104..2a567cce4 100644 --- a/module/plugins/hooks/MultiHoster.py +++ b/module/plugins/hooks/MultiHoster.py @@ -10,8 +10,9 @@ from module.plugins.PluginManager import PluginTuple class MultiHoster(Hook): __version__ = "0.1" + __internal__ = True __description__ = "Gives ability to use MultiHoster services. You need to add your account first." - __config__ = [("activated", "bool", "Activated", True)] + __config__ = [] __author_mail__ = ("pyLoad Team",) __author_mail__ = ("support@pyload.org",) -- cgit v1.2.3 From 718be218273d0acd96c0b0c4739758302044daad Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Feb 2012 12:32:14 +0000 Subject: Don't fail miserably on hoster's internal server error. --- module/plugins/Hoster.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'module/plugins') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index c30fed412..9f3548350 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -161,7 +161,14 @@ class Hoster(Base): self.pyfile.setStatus("starting") - return self.process(self.pyfile) + try: + return self.process(self.pyfile) + except Exception, e: + # Can't seem to import BadHeader + if e.__class__.__name__ == 'BadHeader' and e.code == 500: + self.logInfo("Internal Server Error") + self.tempOffline() + raise e def process(self, pyfile): -- cgit v1.2.3 From 526c6cdbdbe1a352333df162e2c1761bb49dabf2 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Feb 2012 12:38:42 +0000 Subject: Add the internal server error messages to the pyfile. --- module/plugins/Hoster.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/plugins') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 9f3548350..1f21a27c8 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -167,6 +167,7 @@ class Hoster(Base): # Can't seem to import BadHeader if e.__class__.__name__ == 'BadHeader' and e.code == 500: self.logInfo("Internal Server Error") + self.pyfile.error = _("Internal Server Error") self.tempOffline() raise e -- cgit v1.2.3 From ebe0e6039d822e9c16a6095dba8691066bc3b466 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Feb 2012 12:56:40 +0000 Subject: Catch internal server errors on the right place. --- module/plugins/Hoster.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 1f21a27c8..05f55ebc8 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -161,16 +161,7 @@ class Hoster(Base): self.pyfile.setStatus("starting") - try: - return self.process(self.pyfile) - except Exception, e: - # Can't seem to import BadHeader - if e.__class__.__name__ == 'BadHeader' and e.code == 500: - self.logInfo("Internal Server Error") - self.pyfile.error = _("Internal Server Error") - self.tempOffline() - raise e - + return self.process(self.pyfile) def process(self, pyfile): """the 'main' method of every plugin, you **have to** overwrite it""" -- cgit v1.2.3 From 31970754997d545f71135dbf474d276deb3b4698 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Feb 2012 13:19:54 +0000 Subject: If the account is not yet aware of `trafficleft`, force an update to the account info. --- module/plugins/Account.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'module/plugins') diff --git a/module/plugins/Account.py b/module/plugins/Account.py index d30f6920c..704299827 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -242,6 +242,8 @@ class Account(Base, AccountInfo): return parseFileSize(string) / 1024 def formatTrafficleft(self): + if self.trafficleft is None: + self.getAccountInfo(force=True) return formatSize(self.trafficleft*1024) def wrongPassword(self): -- cgit v1.2.3 From 0a453a4ae0294910eb8a1076d9f291b78c7e7eb3 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 13 Feb 2012 14:36:21 +0100 Subject: little account, hoster fix --- module/plugins/Account.py | 6 +++--- module/plugins/Hoster.py | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Account.py b/module/plugins/Account.py index 704299827..323c8b545 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -152,11 +152,11 @@ class Account(Base, AccountInfo): return self.core.requestFactory.getRequest(self.__name__, self.cj) def getDownloadSettings(self): - """ Can be overwritten to change download settings. Default is no chunkLimit, multiDL, resumeDownload + """ Can be overwritten to change download settings. Default is no chunkLimit, max dl limit, resumeDownload - :return: (chunkLimit, multiDL, resumeDownload) / (int,bool,bool) + :return: (chunkLimit, limitDL, resumeDownload) / (int, int ,bool) """ - return -1, True, True + return -1, 0, True @lock def getAccountInfo(self, force=False): diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 05f55ebc8..fc9e23132 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -90,7 +90,7 @@ class Hoster(Base): #: Browser instance, see `network.Browser` self.req = self.account.getAccountRequest() # Default: -1, True, True - self.chunkLimit, self.resumeDownload, self.multiDL = self.account.getDownloadSettings() + self.chunkLimit, self.limitDL, self.resumeDownload = self.account.getDownloadSettings() self.premium = self.account.isPremium() else: self.req = self.core.requestFactory.getRequest(self.__name__) @@ -131,7 +131,10 @@ class Hoster(Base): if self.account: limit = self.account.options.get("limitDL", 0) if limit == "": limit = 0 - return int(limit) + if self.limitDL > 0: # a limit is already set, we use the minimum + return min(int(limit), self.limitDL) + else: + return int(limit) else: return self.limitDL -- cgit v1.2.3 From 4df2b77fdf42046fe19bd371be7c7255986b5980 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 6 Mar 2012 13:36:39 +0100 Subject: renamed hooks to addons, new filemanager and database, many new api methods you will loose ALL your LINKS, webinterface will NOT work --- module/plugins/Account.py | 28 +- module/plugins/Addon.py | 203 +++++++++++++ module/plugins/Base.py | 9 +- module/plugins/Crypter.py | 26 +- module/plugins/Hook.py | 176 ----------- module/plugins/Hoster.py | 4 +- module/plugins/MultiHoster.py | 2 +- module/plugins/PluginManager.py | 16 +- module/plugins/addons/CaptchaTrader.py | 159 ++++++++++ module/plugins/addons/ClickAndLoad.py | 89 ++++++ module/plugins/addons/EasybytezCom.py | 32 ++ module/plugins/addons/Ev0InFetcher.py | 87 ++++++ module/plugins/addons/ExternalScripts.py | 118 ++++++++ module/plugins/addons/ExtractArchive.py | 314 ++++++++++++++++++++ module/plugins/addons/HotFolder.py | 85 ++++++ module/plugins/addons/IRCInterface.py | 425 +++++++++++++++++++++++++++ module/plugins/addons/MergeFiles.py | 94 ++++++ module/plugins/addons/MultiHome.py | 82 ++++++ module/plugins/addons/MultiHoster.py | 102 +++++++ module/plugins/addons/MultishareCz.py | 36 +++ module/plugins/addons/Premium4Me.py | 46 +++ module/plugins/addons/RehostTo.py | 39 +++ module/plugins/addons/UpdateManager.py | 199 +++++++++++++ module/plugins/addons/XMPPInterface.py | 276 +++++++++++++++++ module/plugins/addons/__init__.py | 0 module/plugins/hooks/CaptchaTrader.py | 159 ---------- module/plugins/hooks/ClickAndLoad.py | 89 ------ module/plugins/hooks/EasybytezCom.py | 32 -- module/plugins/hooks/Ev0InFetcher.py | 87 ------ module/plugins/hooks/ExternalScripts.py | 118 -------- module/plugins/hooks/ExtractArchive.py | 314 -------------------- module/plugins/hooks/HotFolder.py | 85 ------ module/plugins/hooks/IRCInterface.py | 425 --------------------------- module/plugins/hooks/MergeFiles.py | 94 ------ module/plugins/hooks/MultiHome.py | 82 ------ module/plugins/hooks/MultiHoster.py | 102 ------- module/plugins/hooks/MultishareCz.py | 36 --- module/plugins/hooks/Premium4Me.py | 46 --- module/plugins/hooks/RehostTo.py | 39 --- module/plugins/hooks/UpdateManager.py | 199 ------------- module/plugins/hooks/XMPPInterface.py | 276 ----------------- module/plugins/hooks/__init__.py | 0 module/plugins/internal/AbstractExtractor.py | 2 +- 43 files changed, 2438 insertions(+), 2394 deletions(-) create mode 100644 module/plugins/Addon.py delete mode 100644 module/plugins/Hook.py create mode 100644 module/plugins/addons/CaptchaTrader.py create mode 100644 module/plugins/addons/ClickAndLoad.py create mode 100644 module/plugins/addons/EasybytezCom.py create mode 100644 module/plugins/addons/Ev0InFetcher.py create mode 100644 module/plugins/addons/ExternalScripts.py create mode 100644 module/plugins/addons/ExtractArchive.py create mode 100644 module/plugins/addons/HotFolder.py create mode 100644 module/plugins/addons/IRCInterface.py create mode 100644 module/plugins/addons/MergeFiles.py create mode 100644 module/plugins/addons/MultiHome.py create mode 100644 module/plugins/addons/MultiHoster.py create mode 100644 module/plugins/addons/MultishareCz.py create mode 100644 module/plugins/addons/Premium4Me.py create mode 100644 module/plugins/addons/RehostTo.py create mode 100644 module/plugins/addons/UpdateManager.py create mode 100644 module/plugins/addons/XMPPInterface.py create mode 100644 module/plugins/addons/__init__.py delete mode 100644 module/plugins/hooks/CaptchaTrader.py delete mode 100644 module/plugins/hooks/ClickAndLoad.py delete mode 100644 module/plugins/hooks/EasybytezCom.py delete mode 100644 module/plugins/hooks/Ev0InFetcher.py delete mode 100644 module/plugins/hooks/ExternalScripts.py delete mode 100644 module/plugins/hooks/ExtractArchive.py delete mode 100644 module/plugins/hooks/HotFolder.py delete mode 100644 module/plugins/hooks/IRCInterface.py delete mode 100644 module/plugins/hooks/MergeFiles.py delete mode 100644 module/plugins/hooks/MultiHome.py delete mode 100644 module/plugins/hooks/MultiHoster.py delete mode 100644 module/plugins/hooks/MultishareCz.py delete mode 100644 module/plugins/hooks/Premium4Me.py delete mode 100644 module/plugins/hooks/RehostTo.py delete mode 100644 module/plugins/hooks/UpdateManager.py delete mode 100644 module/plugins/hooks/XMPPInterface.py delete mode 100644 module/plugins/hooks/__init__.py (limited to 'module/plugins') diff --git a/module/plugins/Account.py b/module/plugins/Account.py index 323c8b545..28d1387fd 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -4,7 +4,7 @@ from time import time from traceback import print_exc from threading import RLock -from module.utils import compare_time, formatSize, parseFileSize, lock, from_string +from module.utils import compare_time, format_size, parseFileSize, lock, from_string from module.Api import AccountInfo from module.network.CookieJar import CookieJar @@ -23,11 +23,15 @@ class Account(Base, AccountInfo): fields of AccountInfo ttype, and can be set easily at runtime. """ + # constants for special values + UNKNOWN = -1 + UNLIMITED = -2 + # Default values valid = True - validuntil = None - trafficleft = None - maxtraffic = None + validuntil = -1 + trafficleft = -1 + maxtraffic = -1 premium = True activated = True @@ -39,7 +43,6 @@ class Account(Base, AccountInfo): # known options known_opt = ("time", "limitDL") - def __init__(self, manager, loginname, password, options): Base.__init__(self, manager.core) @@ -231,7 +234,7 @@ class Account(Base, AccountInfo): except: self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) - if 0 < self.validuntil < time(): + if 0 <= self.validuntil < time(): return False if self.trafficleft is 0: # test explicity for 0 return False @@ -244,7 +247,7 @@ class Account(Base, AccountInfo): def formatTrafficleft(self): if self.trafficleft is None: self.getAccountInfo(force=True) - return formatSize(self.trafficleft*1024) + return format_size(self.trafficleft*1024) def wrongPassword(self): raise WrongPassword @@ -257,12 +260,13 @@ class Account(Base, AccountInfo): self.trafficleft = 0 self.scheduleRefresh(30 * 60) - def expired(self, user): - if user in self.infos: - self.logWarning(_("Account %s is expired, checking again in 1h") % user) + def expired(self, user=None): + if user: self.logDebug("Deprecated argument user for .expired()", user) + + self.logWarning(_("Account %s is expired, checking again in 1h") % user) - self.validuntil = time() - 1 - self.scheduleRefresh(60 * 60) + self.validuntil = time() - 1 + self.scheduleRefresh(60 * 60) def scheduleRefresh(self, time=0, force=True): """ add task to refresh account info to sheduler """ diff --git a/module/plugins/Addon.py b/module/plugins/Addon.py new file mode 100644 index 000000000..fe9ae4817 --- /dev/null +++ b/module/plugins/Addon.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from traceback import print_exc + +#from functools import wraps +from module.utils import has_method + +from Base import Base + +def class_name(p): + return p.rpartition(".")[2] + +class Expose(object): + """ Used for decoration to declare rpc services. You can use any arbitrary method """ + def __new__(cls, f, *args, **kwargs): + addonManager.addRPC(class_name(f.__module__), f.func_name, f.func_doc) + return f + +def AddEventListener(event): + """ Used to register method for events. Arguments needs to match parameter of event """ + class _klass(object): + def __new__(cls, f, *args, **kwargs): + addonManager.addEventListener(class_name(f.__module__), f.func_name, event) + return f + return _klass + +class ConfigHandler(object): + """ Register method as config handler. + + Your method signature has to be: + def foo(value=None): + + value will be passed to use your method to set the config. + When value is None your method needs to return an interaction task for configuration. + """ + + def __new__(cls, f, *args, **kwargs): + addonManager.addConfigHandler(class_name(f.__module__), f.func_name) + return f + +def FileHandler(desc, media, package=False): + """ Register Handler for Files or packages. + Depending on package=True the decorated method needs to accept pid or fid as argument + + :param desc: verbose description + :param media: media type for which your method will be used + :param package: True if it works on packages + """ + pass + +def AddonInfo(desc): + """ Called to retrieve information about the current state. + Decorated method must return anything convertable into string. + + :param desc: verbose description + """ + pass + +def threaded(f): + #@wraps(f) + def run(*args,**kwargs): + addonManager.startThread(f, *args, **kwargs) + return run + +class Addon(Base): + """ + Base class for addon plugins. Use @threaded decorator for all longer running task. + + Decorate methods with @Expose, @AddventListener, @ConfigHandler + + """ + + #: automatically register event listeners for functions, attribute will be deleted dont use it yourself + event_map = None + + # Alternative to event_map + #: List of events the plugin can handle, name the functions exactly like eventname. + event_list = None # dont make duplicate entries in event_map + + #: periodic call interval in secondc + interval = 60 + + def __init__(self, core, manager): + Base.__init__(self, core) + + #: Provide information in dict here, usable by API `getInfo` + self.info = None + + #: Callback of periodical job task, used by addonmanager + self.cb = None + + #: `AddonManager` + self.manager = manager + + #register events + if self.event_map: + for event, funcs in self.event_map.iteritems(): + if type(funcs) in (list, tuple): + for f in funcs: + self.evm.addEvent(event, getattr(self,f)) + else: + self.evm.addEvent(event, getattr(self,funcs)) + + #delete for various reasons + self.event_map = None + + if self.event_list: + for f in self.event_list: + self.evm.addEvent(f, getattr(self,f)) + + self.event_list = None + + self.initPeriodical() + self.init() + self.setup() + + def initPeriodical(self): + if self.interval >=1: + self.cb = self.core.scheduler.addJob(0, self._periodical, threaded=False) + + def _periodical(self): + try: + if self.isActivated(): self.periodical() + except Exception, e: + self.core.log.error(_("Error executing addons: %s") % str(e)) + if self.core.debug: + print_exc() + + self.cb = self.core.scheduler.addJob(self.interval, self._periodical, threaded=False) + + + def __repr__(self): + return "" % self.__name__ + + def isActivated(self): + """ checks if addon is activated""" + return True if self.__internal__ else self.getConfig("activated") + + def init(self): + pass + + def setup(self): + """ more init stuff if needed """ + pass + + def activate(self): + """ Used to activate the addon """ + if has_method(self.__class__, "coreReady"): + self.logDebug("Deprecated method .coreReady() use activate() instead") + self.coreReady() + + def deactivate(self): + """ Used to deactivate the addon. """ + pass + + def periodical(self): + pass + + def newInteractionTask(self, task): + """ new interaction task for the plugin, it MUST set the handler and timeout or will be ignored """ + pass + + def taskCorrect(self, task): + pass + + def taskInvalid(self, task): + pass + + # public events starts from here + def downloadPreparing(self, pyfile): + pass + + def downloadFinished(self, pyfile): + pass + + def downloadFailed(self, pyfile): + pass + + def packageFinished(self, pypack): + pass + + def beforeReconnecting(self, ip): + pass + + def afterReconnecting(self, ip): + pass \ No newline at end of file diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 29ff3a723..b846bbd60 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -39,7 +39,7 @@ class Base(object): __version__ = "0.1" #: Regexp pattern which will be matched for download/crypter plugins __pattern__ = r"" - #: Internal Hook plugin which is always loaded + #: Internal addon plugin which is always loaded __internal__ = False #: Config definition: list of (name, type, verbose_name, default_value) or #: (name, type, verbose_name, short_description, default_value) @@ -50,7 +50,9 @@ class Base(object): __long_description__ = """""" #: List of needed modules __dependencies__ = tuple() - #: Tags to categorize the plugin + #: Used to assign a category to addon plugins + __category__ = "" + #: Tags to categorize the plugin, see documentation for further info __tags__ = tuple() #: Base64 encoded .png icon, please don't use sizes above ~3KB __icon__ = "" @@ -79,7 +81,7 @@ class Base(object): #: :class:`EventManager` self.evm = core.eventManager #: :class:`InteractionManager` - self.im = core.interActionManager + self.im = core.interactionManager def logInfo(self, *args, **kwargs): """ Print args to log at specific level @@ -104,7 +106,6 @@ class Base(object): else: sep = " | " - strings = [] for obj in args: if type(obj) == unicode: diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index 6079ae8f6..15feea8e0 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -2,7 +2,6 @@ from traceback import print_exc -from module.Api import Destination from module.common.packagetools import parseNames from module.utils import to_list, has_method, uniqify from module.utils.fs import exists, remove, fs_encode @@ -11,22 +10,33 @@ from Base import Base, Retry class Package: """ Container that indicates new package should be created """ - def __init__(self, name, urls=None, dest=Destination.Queue): + def __init__(self, name, urls=None): self.name = name self.urls = urls if urls else [] - self.dest = dest + # nested packages + self.packs = [] - def addUrl(self, url): + def addURL(self, url): self.urls.append(url) + def addPackage(self, pack): + self.packs.append(pack) + + def getAllURLs(self): + urls = self.urls + for p in self.packs: + urls.extend(p.getAllURLs()) + return urls + + # same name and urls is enough to be equal for packages def __eq__(self, other): return self.name == other.name and self.urls == other.urls def __repr__(self): - return u". - - @author: RaNaN -""" - -from traceback import print_exc - -#from functools import wraps -from module.utils import has_method - -from Base import Base - -def class_name(p): - return p.rpartition(".")[2] - -class Expose(object): - """ used for decoration to declare rpc services """ - def __new__(cls, f, *args, **kwargs): - hookManager.addRPC(class_name(f.__module__), f.func_name, f.func_doc) - return f - -def AddEventListener(event): - """ used to register method for events """ - class _klass(object): - def __new__(cls, f, *args, **kwargs): - hookManager.addEventListener(class_name(f.__module__), f.func_name, event) - return f - return _klass - - -class ConfigHandler(object): - """ register method as config handler """ - def __new__(cls, f, *args, **kwargs): - hookManager.addConfigHandler(class_name(f.__module__), f.func_name) - return f - -def threaded(f): - #@wraps(f) - def run(*args,**kwargs): - hookManager.startThread(f, *args, **kwargs) - return run - -class Hook(Base): - """ - Base class for hook plugins. Please use @threaded decorator for all longer running task. - """ - - #: automatically register event listeners for functions, attribute will be deleted dont use it yourself - event_map = None - - # Alternative to event_map - #: List of events the plugin can handle, name the functions exactly like eventname. - event_list = None # dont make duplicate entries in event_map - - - #: periodic call interval in secondc - interval = 60 - - def __init__(self, core, manager): - Base.__init__(self, core) - - #: Provide information in dict here, usable by API `getInfo` - self.info = None - - #: Callback of periodical job task, used by hookmanager - self.cb = None - - #: `HookManager` - self.manager = manager - - #register events - if self.event_map: - for event, funcs in self.event_map.iteritems(): - if type(funcs) in (list, tuple): - for f in funcs: - self.evm.addEvent(event, getattr(self,f)) - else: - self.evm.addEvent(event, getattr(self,funcs)) - - #delete for various reasons - self.event_map = None - - if self.event_list: - for f in self.event_list: - self.evm.addEvent(f, getattr(self,f)) - - self.event_list = None - - self.initPeriodical() - self.init() - self.setup() - - def initPeriodical(self): - if self.interval >=1: - self.cb = self.core.scheduler.addJob(0, self._periodical, threaded=False) - - def _periodical(self): - try: - if self.isActivated(): self.periodical() - except Exception, e: - self.core.log.error(_("Error executing hooks: %s") % str(e)) - if self.core.debug: - print_exc() - - self.cb = self.core.scheduler.addJob(self.interval, self._periodical, threaded=False) - - - def __repr__(self): - return "" % self.__name__ - - def isActivated(self): - """ checks if hook is activated""" - return True if self.__internal__ else self.getConfig("activated") - - def init(self): - pass - - def setup(self): - """ more init stuff if needed """ - pass - - def activate(self): - """ Used to activate the hook """ - if has_method(self.__class__, "coreReady"): - self.logDebug("Deprecated method .coreReady() use activated() instead") - self.coreReady() - - def deactivate(self): - """ Used to deactivate the hook. """ - pass - - def periodical(self): - pass - - def newCaptchaTask(self, task): - """ new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """ - pass - - def captchaCorrect(self, task): - pass - - def captchaInvalid(self, task): - pass - - # public events starts from here - def downloadPreparing(self, pyfile): - pass - - def downloadFinished(self, pyfile): - pass - - def downloadFailed(self, pyfile): - pass - - def packageFinished(self, pypack): - pass - - def beforeReconnecting(self, ip): - pass - - def afterReconnecting(self, ip): - pass \ No newline at end of file diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index fc9e23132..32c587aa5 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -185,7 +185,7 @@ class Hoster(Base): 10 - not implemented 20 - unknown error """ - #@TODO checksum check hook + #@TODO checksum check addon return True, 10 @@ -365,7 +365,7 @@ class Hoster(Base): filename = join(location, name) - self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) + self.core.addonManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, diff --git a/module/plugins/MultiHoster.py b/module/plugins/MultiHoster.py index abbc14466..1936478b4 100644 --- a/module/plugins/MultiHoster.py +++ b/module/plugins/MultiHoster.py @@ -15,7 +15,7 @@ class MultiHoster(Account): """ Base class for MultiHoster services. This is also an Account instance so you should see :class:`Account` and overwrite necessary methods. - Multihoster becomes only active when an Account was entered and the MultiHoster hook was activated. + Multihoster becomes only active when an Account was entered and the MultiHoster addon was activated. You need to overwrite `loadHosterList` and a corresponding :class:`Hoster` plugin with the same name should be available to make your service working. """ diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index 4e2fa21ed..733cd2c5d 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -30,9 +30,11 @@ from module.plugins.Base import Base from new_collections import namedtuple +#TODO: ignores not updatable + # ignore these plugin configs, mainly because plugins were wiped out IGNORE = ( - "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('hooks', 'UnRar'), + "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('addons', 'UnRar'), 'EasyShareCom', 'FlyshareCz' ) @@ -41,7 +43,7 @@ PluginTuple = namedtuple("PluginTuple", "version re deps user path") class PluginManager: ROOT = "module.plugins." USERROOT = "userplugins." - TYPES = ("crypter", "hoster", "captcha", "accounts", "hooks", "internal") + TYPES = ("crypter", "hoster", "accounts", "addons", "internal") BUILTIN = re.compile(r'__(?P[a-z0-9_]+)__\s*=\s?(True|False|None|[0-9x.]+)', re.I) SINGLE = re.compile(r'__(?P[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?. + + @author: mkaay, RaNaN +""" + +try: + from json import loads +except ImportError: + from simplejson import loads + +from thread import start_new_thread +from pycurl import FORM_FILE, LOW_SPEED_TIME + +from module.network.RequestFactory import getURL, getRequest +from module.network.HTTPRequest import BadHeader + +from module.plugins.Addon import Addon + +PYLOAD_KEY = "9f65e7f381c3af2b076ea680ae96b0b7" + +class CaptchaTraderException(Exception): + def __init__(self, err): + self.err = err + + def getCode(self): + return self.err + + def __str__(self): + return "" % self.err + + def __repr__(self): + return "" % self.err + +class CaptchaTrader(Addon): + __name__ = "CaptchaTrader" + __version__ = "0.13" + __description__ = """send captchas to captchatrader.com""" + __config__ = [("activated", "bool", "Activated", True), + ("username", "str", "Username", ""), + ("force", "bool", "Force CT even if client is connected", False), + ("passkey", "password", "Password", ""),] + __author_name__ = ("RaNaN") + __author_mail__ = ("RaNaN@pyload.org") + + SUBMIT_URL = "http://captchatrader.com/api/submit" + RESPOND_URL = "http://captchatrader.com/api/respond" + GETCREDITS_URL = "http://captchatrader.com/api/get_credits/username:%(user)s/password:%(password)s/" + + def setup(self): + self.info = {} + + def getCredits(self): + json = getURL(CaptchaTrader.GETCREDITS_URL % {"user": self.getConfig("username"), + "password": self.getConfig("passkey")}) + response = loads(json) + if response[0] < 0: + raise CaptchaTraderException(response[1]) + else: + self.logInfo(_("%s credits left") % response[1]) + self.info["credits"] = response[1] + return response[1] + + def submit(self, captcha, captchaType="file", match=None): + if not PYLOAD_KEY: + raise CaptchaTraderException("No API Key Specified!") + + #if type(captcha) == str and captchaType == "file": + # raise CaptchaTraderException("Invalid Type") + assert captchaType in ("file", "url-jpg", "url-jpeg", "url-png", "url-bmp") + + req = getRequest() + + #raise timeout threshold + req.c.setopt(LOW_SPEED_TIME, 80) + + try: + json = req.load(CaptchaTrader.SUBMIT_URL, post={"api_key": PYLOAD_KEY, + "username": self.getConfig("username"), + "password": self.getConfig("passkey"), + "value": (FORM_FILE, captcha), + "type": captchaType}, multipart=True) + finally: + req.close() + + response = loads(json) + if response[0] < 0: + raise CaptchaTraderException(response[1]) + + ticket = response[0] + result = response[1] + self.logDebug("result %s : %s" % (ticket,result)) + + return ticket, result + + def respond(self, ticket, success): + try: + json = getURL(CaptchaTrader.RESPOND_URL, post={"is_correct": 1 if success else 0, + "username": self.getConfig("username"), + "password": self.getConfig("passkey"), + "ticket": ticket}) + + response = loads(json) + if response[0] < 0: + raise CaptchaTraderException(response[1]) + + except BadHeader, e: + self.logError(_("Could not send response."), str(e)) + + def newCaptchaTask(self, task): + if not task.isTextual(): + return False + + if not self.getConfig("username") or not self.getConfig("passkey"): + return False + + if self.core.isClientConnected() and not self.getConfig("force"): + return False + + if self.getCredits() > 10: + task.handler.append(self) + task.setWaiting(100) + start_new_thread(self.processCaptcha, (task,)) + + else: + self.logInfo(_("Your CaptchaTrader Account has not enough credits")) + + def captchaCorrect(self, task): + if "ticket" in task.data: + ticket = task.data["ticket"] + self.respond(ticket, True) + + def captchaInvalid(self, task): + if "ticket" in task.data: + ticket = task.data["ticket"] + self.respond(ticket, False) + + def processCaptcha(self, task): + c = task.captchaFile + try: + ticket, result = self.submit(c) + except CaptchaTraderException, e: + task.error = e.getCode() + return + + task.data["ticket"] = ticket + task.setResult(result) diff --git a/module/plugins/addons/ClickAndLoad.py b/module/plugins/addons/ClickAndLoad.py new file mode 100644 index 000000000..6d6928557 --- /dev/null +++ b/module/plugins/addons/ClickAndLoad.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN + @interface-version: 0.2 +""" + +import socket +import thread + +from module.plugins.Addon import Addon + +class ClickAndLoad(Addon): + __name__ = "ClickAndLoad" + __version__ = "0.2" + __description__ = """Gives abillity to use jd's click and load. depends on webinterface""" + __config__ = [("activated", "bool", "Activated", "True"), + ("extern", "bool", "Allow external link adding", "False")] + __author_name__ = ("RaNaN", "mkaay") + __author_mail__ = ("RaNaN@pyload.de", "mkaay@mkaay.de") + + def activate(self): + self.port = int(self.core.config['webinterface']['port']) + if self.core.config['webinterface']['activated']: + try: + if self.getConfig("extern"): + ip = "0.0.0.0" + else: + ip = "127.0.0.1" + + thread.start_new_thread(proxy, (self, ip, self.port, 9666)) + except: + self.log.error("ClickAndLoad port already in use.") + + +def proxy(self, *settings): + thread.start_new_thread(server, (self,) + settings) + lock = thread.allocate_lock() + lock.acquire() + lock.acquire() + + +def server(self, *settings): + try: + dock_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + dock_socket.bind((settings[0], settings[2])) + dock_socket.listen(5) + while True: + client_socket = dock_socket.accept()[0] + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.connect(("127.0.0.1", settings[1])) + thread.start_new_thread(forward, (client_socket, server_socket)) + thread.start_new_thread(forward, (server_socket, client_socket)) + except socket.error, e: + if hasattr(e, "errno"): + errno = e.errno + else: + errno = e.args[0] + + if errno == 98: + self.core.log.warning(_("Click'N'Load: Port 9666 already in use")) + return + thread.start_new_thread(server, (self,) + settings) + except: + thread.start_new_thread(server, (self,) + settings) + + +def forward(source, destination): + string = ' ' + while string: + string = source.recv(1024) + if string: + destination.sendall(string) + else: + #source.shutdown(socket.SHUT_RD) + destination.shutdown(socket.SHUT_WR) diff --git a/module/plugins/addons/EasybytezCom.py b/module/plugins/addons/EasybytezCom.py new file mode 100644 index 000000000..4dd39cca6 --- /dev/null +++ b/module/plugins/addons/EasybytezCom.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +from module.network.RequestFactory import getURL +from module.plugins.internal.MultiHoster import MultiHoster +import re + +def getConfigSet(option): + s = set(option.lower().replace(',','|').split('|')) + s.discard(u'') + return s + +class EasybytezCom(MultiHoster): + __name__ = "EasybytezCom" + __version__ = "0.01" + __type__ = "hook" + __config__ = [("activated", "bool", "Activated", "False"), + ("includeHoster", "str", "Use only for downloads from (comma-separated hosters)", ""), + ("excludeHoster", "str", "Do not use for downloads from (comma-separated hosters)", "")] + __description__ = """EasyBytez.com hook plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + def getHoster(self): + + hoster = set(['2shared.com', 'easy-share.com', 'filefactory.com', 'fileserve.com', 'filesonic.com', 'hotfile.com', 'mediafire.com', 'megaupload.com', 'netload.in', 'rapidshare.com', 'uploading.com', 'wupload.com', 'oron.com', 'uploadstation.com', 'ul.to', 'uploaded.to']) + + option = self.getConfig('includeHoster').strip() + if option: hoster &= getConfigSet(option) + option = self.getConfig('excludeHoster').strip() + if option: hoster -= getConfigSet(option) + + return list(hoster) \ No newline at end of file diff --git a/module/plugins/addons/Ev0InFetcher.py b/module/plugins/addons/Ev0InFetcher.py new file mode 100644 index 000000000..aeb46320a --- /dev/null +++ b/module/plugins/addons/Ev0InFetcher.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" +from module.lib import feedparser +from time import mktime, time + +from module.plugins.Addon import Addon + +class Ev0InFetcher(Addon): + __name__ = "Ev0InFetcher" + __version__ = "0.2" + __description__ = """checks rss feeds for ev0.in""" + __config__ = [("activated", "bool", "Activated", "False"), + ("interval", "int", "Check interval in minutes", "10"), + ("queue", "bool", "Move new shows directly to Queue", False), + ("shows", "str", "Shows to check for (comma seperated)", ""), + ("quality", "xvid;x264;rmvb", "Video Format", "xvid"), + ("hoster", "str", "Hoster to use (comma seperated)", "NetloadIn,RapidshareCom,MegauploadCom,HotfileCom")] + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") + + def setup(self): + self.interval = self.getConfig("interval") * 60 + + def filterLinks(self, links): + results = self.core.pluginManager.parseUrls(links) + sortedLinks = {} + + for url, hoster in results[0]: + if hoster not in sortedLinks: + sortedLinks[hoster] = [] + sortedLinks[hoster].append(url) + + for h in self.getConfig("hoster").split(","): + try: + return sortedLinks[h.strip()] + except: + continue + return [] + + def periodical(self): + def normalizefiletitle(filename): + filename = filename.replace('.', ' ') + filename = filename.replace('_', ' ') + filename = filename.lower() + return filename + + shows = [s.strip() for s in self.getConfig("shows").split(",")] + + feed = feedparser.parse("http://feeds.feedburner.com/ev0in/%s?format=xml" % self.getConfig("quality")) + + showStorage = {} + for show in shows: + showStorage[show] = int(self.getStorage("show_%s_lastfound" % show, 0)) + + found = False + for item in feed['items']: + for show, lastfound in showStorage.iteritems(): + if show.lower() in normalizefiletitle(item['title']) and lastfound < int(mktime(item.date_parsed)): + links = self.filterLinks(item['description'].split("
")) + packagename = item['title'].encode("utf-8") + self.core.log.info("Ev0InFetcher: new episode '%s' (matched '%s')" % (packagename, show)) + self.core.api.addPackage(packagename, links, 1 if self.getConfig("queue") else 0) + self.setStorage("show_%s_lastfound" % show, int(mktime(item.date_parsed))) + found = True + if not found: + #self.core.log.debug("Ev0InFetcher: no new episodes found") + pass + + for show, lastfound in self.getStorage().iteritems(): + if int(lastfound) > 0 and int(lastfound) + (3600*24*30) < int(time()): + self.delStorage("show_%s_lastfound" % show) + self.core.log.debug("Ev0InFetcher: cleaned '%s' record" % show) diff --git a/module/plugins/addons/ExternalScripts.py b/module/plugins/addons/ExternalScripts.py new file mode 100644 index 000000000..00fc7c114 --- /dev/null +++ b/module/plugins/addons/ExternalScripts.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +import subprocess +from os import access, X_OK, makedirs +from os.path import basename + +from module.plugins.Addon import Addon +from module.utils.fs import save_join, exists, join, listdir + +class ExternalScripts(Addon): + __name__ = "ExternalScripts" + __version__ = "0.21" + __description__ = """Run external scripts""" + __config__ = [("activated", "bool", "Activated", "True")] + __author_name__ = ("mkaay", "RaNaN", "spoob") + __author_mail__ = ("mkaay@mkaay.de", "ranan@pyload.org", "spoob@pyload.org") + + event_list = ["unrarFinished", "allDownloadsFinished", "allDownloadsProcessed"] + + def setup(self): + self.scripts = {} + + folders = ['download_preparing', 'download_finished', 'package_finished', + 'before_reconnect', 'after_reconnect', 'unrar_finished', + 'all_dls_finished', 'all_dls_processed'] + + for folder in folders: + + self.scripts[folder] = [] + + self.initPluginType(folder, join(pypath, 'scripts', folder)) + self.initPluginType(folder, join('scripts', folder)) + + for script_type, names in self.scripts.iteritems(): + if names: + self.logInfo((_("Installed scripts for %s: ") % script_type ) + ", ".join([basename(x) for x in names])) + + + def initPluginType(self, folder, path): + if not exists(path): + try: + makedirs(path) + except : + self.logDebug("Script folder %s not created" % folder) + return + + for f in listdir(path): + if f.startswith("#") or f.startswith(".") or f.startswith("_") or f.endswith("~") or f.endswith(".swp"): + continue + + if not access(join(path,f), X_OK): + self.logWarning(_("Script not executable:") + " %s/%s" % (folder, f)) + + self.scripts[folder].append(join(path, f)) + + def callScript(self, script, *args): + try: + cmd = [script] + [str(x) if not isinstance(x, basestring) else x for x in args] + #output goes to pyload + subprocess.Popen(cmd, bufsize=-1) + except Exception, e: + self.logError(_("Error in %(script)s: %(error)s") % { "script" :basename(script), "error": str(e)}) + + def downloadPreparing(self, pyfile): + for script in self.scripts['download_preparing']: + self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.id) + + def downloadFinished(self, pyfile): + for script in self.scripts['download_finished']: + self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.name, pyfile.id, + save_join(self.core.config['general']['download_folder'], pyfile.package().folder, pyfile.name), + pyfile.id) + + + def packageFinished(self, pypack): + for script in self.scripts['package_finished']: + folder = self.core.config['general']['download_folder'] + folder = save_join(folder, pypack.folder) + + self.callScript(script, pypack.name, folder, pypack.id) + + def beforeReconnecting(self, ip): + for script in self.scripts['before_reconnect']: + self.callScript(script, ip) + + def afterReconnecting(self, ip): + for script in self.scripts['after_reconnect']: + self.callScript(script, ip) + + def unrarFinished(self, folder, fname): + for script in self.scripts["unrar_finished"]: + self.callScript(script, folder, fname) + + def allDownloadsFinished(self): + for script in self.scripts["all_dls_finished"]: + self.callScript(script) + + def allDownloadsProcessed(self): + for script in self.scripts["all_dls_processed"]: + self.callScript(script) + diff --git a/module/plugins/addons/ExtractArchive.py b/module/plugins/addons/ExtractArchive.py new file mode 100644 index 000000000..5f749ed0d --- /dev/null +++ b/module/plugins/addons/ExtractArchive.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os +from os.path import basename, isfile, isdir, join +from traceback import print_exc +from copy import copy + +# monkey patch bug in python 2.6 and lower +# see http://bugs.python.org/issue6122 +# http://bugs.python.org/issue1236 +# http://bugs.python.org/issue1731717 +if sys.version_info < (2, 7) and os.name != "nt": + from subprocess import Popen + import errno + + def _eintr_retry_call(func, *args): + while True: + try: + return func(*args) + except OSError, e: + if e.errno == errno.EINTR: + continue + raise + + # unsued timeout option for older python version + def wait(self, timeout=0): + """Wait for child process to terminate. Returns returncode + attribute.""" + if self.returncode is None: + try: + pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) + except OSError, e: + if e.errno != errno.ECHILD: + raise + # This happens if SIGCLD is set to be ignored or waiting + # for child processes has otherwise been disabled for our + # process. This child is dead, we can't get the status. + sts = 0 + self._handle_exitstatus(sts) + return self.returncode + + Popen.wait = wait + +if os.name != "nt": + from os import chown + from pwd import getpwnam + from grp import getgrnam + +from module.utils.fs import save_join, fs_encode, exists, remove, chmod, makedirs +from module.plugins.Addon import Addon, threaded, Expose +from module.plugins.internal.AbstractExtractor import ArchiveError, CRCError, WrongPassword + +class ExtractArchive(Addon): + """ + Provides: unrarFinished (folder, filename) + """ + __name__ = "ExtractArchive" + __version__ = "0.12" + __description__ = "Extract different kind of archives" + __config__ = [("activated", "bool", "Activated", True), + ("fullpath", "bool", "Extract full path", True), + ("overwrite", "bool", "Overwrite files", True), + ("passwordfile", "file", "password file", "unrar_passwords.txt"), + ("deletearchive", "bool", "Delete archives when done", False), + ("subfolder", "bool", "Create subfolder for each package", False), + ("destination", "folder", "Extract files to", ""), + ("recursive", "bool", "Extract archives in archvies", True), + ("queue", "bool", "Wait for all downloads to be finished", True), + ("renice", "int", "CPU Priority", 0), ] + __author_name__ = ("pyload Team") + __author_mail__ = ("adminpyload.org") + + event_list = ["allDownloadsProcessed"] + + def setup(self): + self.plugins = [] + self.passwords = [] + names = [] + + for p in ("UnRar", "UnZip"): + try: + module = self.core.pluginManager.loadModule("internal", p) + klass = getattr(module, p) + if klass.checkDeps(): + names.append(p) + self.plugins.append(klass) + + except OSError, e: + if e.errno == 2: + self.logInfo(_("No %s installed") % p) + else: + self.logWarning(_("Could not activate %s") % p, str(e)) + if self.core.debug: + print_exc() + + except Exception, e: + self.logWarning(_("Could not activate %s") % p, str(e)) + if self.core.debug: + print_exc() + + if names: + self.logInfo(_("Activated") + " " + " ".join(names)) + else: + self.logInfo(_("No Extract plugins activated")) + + # queue with package ids + self.queue = [] + + @Expose + def extractPackage(self, id): + """ Extract package with given id""" + self.manager.startThread(self.extract, [id]) + + def packageFinished(self, pypack): + if self.getConfig("queue"): + self.logInfo(_("Package %s queued for later extracting") % pypack.name) + self.queue.append(pypack.id) + else: + self.manager.startThread(self.extract, [pypack.id]) + + + @threaded + def allDownloadsProcessed(self, thread): + local = copy(self.queue) + del self.queue[:] + self.extract(local, thread) + + + def extract(self, ids, thread=None): + # reload from txt file + self.reloadPasswords() + + # dl folder + dl = self.config['general']['download_folder'] + + extracted = [] + + #iterate packages -> plugins -> targets + for pid in ids: + p = self.core.files.getPackage(pid) + self.logInfo(_("Check package %s") % p.name) + if not p: continue + + # determine output folder + out = save_join(dl, p.folder, "") + # force trailing slash + + if self.getConfig("destination") and self.getConfig("destination").lower() != "none": + + out = save_join(dl, p.folder, self.getConfig("destination"), "") + #relative to package folder if destination is relative, otherwise absolute path overwrites them + + if self.getConf("subfolder"): + out = join(out, fs_encode(p.folder)) + + if not exists(out): + makedirs(out) + + files_ids = [(save_join(dl, p.folder, x["name"]), x["id"]) for x in p.getChildren().itervalues()] + matched = False + + # check as long there are unseen files + while files_ids: + new_files_ids = [] + + for plugin in self.plugins: + targets = plugin.getTargets(files_ids) + if targets: + self.logDebug("Targets for %s: %s" % (plugin.__name__, targets)) + matched = True + for target, fid in targets: + if target in extracted: + self.logDebug(basename(target), "skipped") + continue + extracted.append(target) #prevent extracting same file twice + + klass = plugin(self, target, out, self.getConfig("fullpath"), self.getConfig("overwrite"), + self.getConfig("renice")) + klass.init() + + self.logInfo(basename(target), _("Extract to %s") % out) + new_files = self.startExtracting(klass, fid, p.password.strip().splitlines(), thread) + self.logDebug("Extracted: %s" % new_files) + self.setPermissions(new_files) + + for file in new_files: + if not exists(file): + self.logDebug("new file %s does not exists" % file) + continue + if self.getConfig("recursive") and isfile(file): + new_files_ids.append((file, fid)) #append as new target + + files_ids = new_files_ids # also check extracted files + + if not matched: self.logInfo(_("No files found to extract")) + + + + def startExtracting(self, plugin, fid, passwords, thread): + pyfile = self.core.files.getFile(fid) + if not pyfile: return [] + + pyfile.setCustomStatus(_("extracting")) + thread.addActive(pyfile) #keep this file until everything is done + + try: + progress = lambda x: pyfile.setProgress(x) + success = False + + if not plugin.checkArchive(): + plugin.extract(progress) + success = True + else: + self.logInfo(basename(plugin.file), _("Password protected")) + self.logDebug("Passwords: %s" % str(passwords)) + + pwlist = copy(self.getPasswords()) + #remove already supplied pws from list (only local) + for pw in passwords: + if pw in pwlist: pwlist.remove(pw) + + for pw in passwords + pwlist: + try: + self.logDebug("Try password: %s" % pw) + if plugin.checkPassword(pw): + plugin.extract(progress, pw) + self.addPassword(pw) + success = True + break + except WrongPassword: + self.logDebug("Password was wrong") + + if not success: + self.logError(basename(plugin.file), _("Wrong password")) + return [] + + if self.core.debug: + self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles())) + + if self.getConfig("deletearchive"): + files = plugin.getDeleteFiles() + self.logInfo(_("Deleting %s files") % len(files)) + for f in files: + if exists(f): remove(f) + else: self.logDebug("%s does not exists" % f) + + self.logInfo(basename(plugin.file), _("Extracting finished")) + self.manager.dispatchEvent("unrarFinished", plugin.out, plugin.file) + + return plugin.getExtractedFiles() + + + except ArchiveError, e: + self.logError(basename(plugin.file), _("Archive Error"), str(e)) + except CRCError: + self.logError(basename(plugin.file), _("CRC Mismatch")) + except Exception, e: + if self.core.debug: + print_exc() + self.logError(basename(plugin.file), _("Unknown Error"), str(e)) + + return [] + + @Expose + def getPasswords(self): + """ List of saved passwords """ + return self.passwords + + + def reloadPasswords(self): + pwfile = self.getConfig("passwordfile") + if not exists(pwfile): + open(pwfile, "wb").close() + + passwords = [] + f = open(pwfile, "rb") + for pw in f.read().splitlines(): + passwords.append(pw) + f.close() + + self.passwords = passwords + + + @Expose + def addPassword(self, pw): + """ Adds a password to saved list""" + pwfile = self.getConfig("passwordfile") + + if pw in self.passwords: self.passwords.remove(pw) + self.passwords.insert(0, pw) + + f = open(pwfile, "wb") + for pw in self.passwords: + f.write(pw + "\n") + f.close() + + def setPermissions(self, files): + for f in files: + if not exists(f): continue + try: + if self.core.config["permission"]["change_file"]: + if isfile(f): + chmod(f, int(self.core.config["permission"]["file"], 8)) + elif isdir(f): + chmod(f, int(self.core.config["permission"]["folder"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + chown(f, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed"), e) diff --git a/module/plugins/addons/HotFolder.py b/module/plugins/addons/HotFolder.py new file mode 100644 index 000000000..d05026448 --- /dev/null +++ b/module/plugins/addons/HotFolder.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN + @interface-version: 0.2 +""" + +from os import makedirs +from os import listdir +from os.path import exists +from os.path import join +from os.path import isfile +from shutil import move +import time + +from module.plugins.Addon import Addon + +class HotFolder(Addon): + __name__ = "HotFolder" + __version__ = "0.1" + __description__ = """observe folder and file for changes and add container and links""" + __config__ = [ ("activated", "bool", "Activated" , "False"), + ("folder", "str", "Folder to observe", "container"), + ("watch_file", "bool", "Observe link file", "False"), + ("keep", "bool", "Keep added containers", "True"), + ("file", "str", "Link file", "links.txt")] + __threaded__ = [] + __author_name__ = ("RaNaN") + __author_mail__ = ("RaNaN@pyload.de") + + def setup(self): + self.interval = 10 + + def periodical(self): + + if not exists(join(self.getConfig("folder"), "finished")): + makedirs(join(self.getConfig("folder"), "finished")) + + if self.getConfig("watch_file"): + + if not exists(self.getConfig("file")): + f = open(self.getConfig("file"), "wb") + f.close() + + + f = open(self.getConfig("file"), "rb") + content = f.read().strip() + f.close() + f = open(self.getConfig("file"), "wb") + f.close() + if content: + name = "%s_%s.txt" % (self.getConfig("file"), time.strftime("%H-%M-%S_%d%b%Y") ) + + f = open(join(self.getConfig("folder"), "finished", name), "wb") + f.write(content) + f.close() + + self.core.api.addPackage(f.name, [f.name], 1) + + for f in listdir(self.getConfig("folder")): + path = join(self.getConfig("folder"), f) + + if not isfile(path) or f.endswith("~") or f.startswith("#") or f.startswith("."): + continue + + newpath = join(self.getConfig("folder"), "finished", f if self.getConfig("keep") else "tmp_"+f) + move(path, newpath) + + self.log.info(_("Added %s from HotFolder") % f) + self.core.api.addPackage(f, [newpath], 1) + + \ No newline at end of file diff --git a/module/plugins/addons/IRCInterface.py b/module/plugins/addons/IRCInterface.py new file mode 100644 index 000000000..ddaa40613 --- /dev/null +++ b/module/plugins/addons/IRCInterface.py @@ -0,0 +1,425 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN + @author: jeix + @interface-version: 0.2 +""" + +from select import select +import socket +from threading import Thread +import time +from time import sleep +from traceback import print_exc +import re + +from module.plugins.Addon import Addon +from module.network.RequestFactory import getURL +from module.utils import formatSize + +from pycurl import FORM_FILE + +class IRCInterface(Thread, Addon): + __name__ = "IRCInterface" + __version__ = "0.1" + __description__ = """connect to irc and let owner perform different tasks""" + __config__ = [("activated", "bool", "Activated", "False"), + ("host", "str", "IRC-Server Address", "Enter your server here!"), + ("port", "int", "IRC-Server Port", "6667"), + ("ident", "str", "Clients ident", "pyload-irc"), + ("realname", "str", "Realname", "pyload-irc"), + ("nick", "str", "Nickname the Client will take", "pyLoad-IRC"), + ("owner", "str", "Nickname the Client will accept commands from", "Enter your nick here!"), + ("info_file", "bool", "Inform about every file finished", "False"), + ("info_pack", "bool", "Inform about every package finished", "True"), + ("captcha", "bool", "Send captcha requests", "True")] + __author_name__ = ("Jeix") + __author_mail__ = ("Jeix@hasnomail.com") + + def __init__(self, core, manager): + Thread.__init__(self) + Addon.__init__(self, core, manager) + self.setDaemon(True) + # self.sm = core.server_methods + self.api = core.api #todo, only use api + + def coreReady(self): + self.new_package = {} + + self.abort = False + + self.links_added = 0 + self.more = [] + + self.start() + + + def packageFinished(self, pypack): + try: + if self.getConfig("info_pack"): + self.response(_("Package finished: %s") % pypack.name) + except: + pass + + def downloadFinished(self, pyfile): + try: + if self.getConfig("info_file"): + self.response(_("Download finished: %(name)s @ %(plugin)s ") % { "name" : pyfile.name, "plugin": pyfile.pluginname} ) + except: + pass + + def newCaptchaTask(self, task): + if self.getConfig("captcha") and task.isTextual(): + task.handler.append(self) + task.setWaiting(60) + + page = getURL("http://www.freeimagehosting.net/upload.php", post={"attached" : (FORM_FILE, task.captchaFile)}, multipart=True) + + url = re.search(r"\[img\]([^\[]+)\[/img\]\[/url\]", page).group(1) + self.response(_("New Captcha Request: %s") % url) + self.response(_("Answer with 'c %s text on the captcha'") % task.id) + + def run(self): + # connect to IRC etc. + self.sock = socket.socket() + host = self.getConfig("host") + self.sock.connect((host, self.getConfig("port"))) + nick = self.getConfig("nick") + self.sock.send("NICK %s\r\n" % nick) + self.sock.send("USER %s %s bla :%s\r\n" % (nick, host, nick)) + for t in self.getConfig("owner").split(): + if t.strip().startswith("#"): + self.sock.send("JOIN %s\r\n" % t.strip()) + self.log.info("pyLoad IRC: Connected to %s!" % host) + self.log.info("pyLoad IRC: Switching to listening mode!") + try: + self.main_loop() + + except IRCError, ex: + self.sock.send("QUIT :byebye\r\n") + print_exc() + self.sock.close() + + + def main_loop(self): + readbuffer = "" + while True: + sleep(1) + fdset = select([self.sock], [], [], 0) + if self.sock not in fdset[0]: + continue + + if self.abort: + raise IRCError("quit") + + readbuffer += self.sock.recv(1024) + temp = readbuffer.split("\n") + readbuffer = temp.pop() + + for line in temp: + line = line.rstrip() + first = line.split() + + if first[0] == "PING": + self.sock.send("PONG %s\r\n" % first[1]) + + if first[0] == "ERROR": + raise IRCError(line) + + msg = line.split(None, 3) + if len(msg) < 4: + continue + + msg = { + "origin":msg[0][1:], + "action":msg[1], + "target":msg[2], + "text":msg[3][1:] + } + + self.handle_events(msg) + + + def handle_events(self, msg): + if not msg["origin"].split("!", 1)[0] in self.getConfig("owner").split(): + return + + if msg["target"].split("!", 1)[0] != self.getConfig("nick"): + return + + if msg["action"] != "PRIVMSG": + return + + # HANDLE CTCP ANTI FLOOD/BOT PROTECTION + if msg["text"] == "\x01VERSION\x01": + self.log.debug("Sending CTCP VERSION.") + self.sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface")) + return + elif msg["text"] == "\x01TIME\x01": + self.log.debug("Sending CTCP TIME.") + self.sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time.time())) + return + elif msg["text"] == "\x01LAG\x01": + self.log.debug("Received CTCP LAG.") # don't know how to answer + return + + trigger = "pass" + args = None + + try: + temp = msg["text"].split() + trigger = temp[0] + if len(temp) > 1: + args = temp[1:] + except: + pass + + handler = getattr(self, "event_%s" % trigger, self.event_pass) + try: + res = handler(args) + for line in res: + self.response(line, msg["origin"]) + except Exception, e: + self.log.error("pyLoad IRC: "+ repr(e)) + + + def response(self, msg, origin=""): + if origin == "": + for t in self.getConfig("owner").split(): + self.sock.send("PRIVMSG %s :%s\r\n" % (t.strip(), msg)) + else: + self.sock.send("PRIVMSG %s :%s\r\n" % (origin.split("!", 1)[0], msg)) + + +#### Events + def event_pass(self, args): + return [] + + def event_status(self, args): + downloads = self.api.statusDownloads() + if not downloads: + return ["INFO: There are no active downloads currently."] + + temp_progress = "" + lines = ["ID - Name - Status - Speed - ETA - Progress"] + for data in downloads: + + if data.status == 5: + temp_progress = data.format_wait + else: + temp_progress = "%d%% (%s)" % (data.percent, data.format_size) + + lines.append("#%d - %s - %s - %s - %s - %s" % + ( + data.fid, + data.name, + data.statusmsg, + "%s/s" % formatSize(data.speed), + "%s" % data.format_eta, + temp_progress + ) + ) + return lines + + def event_queue(self, args): + ps = self.api.getQueue() + + if not ps: + return ["INFO: There are no packages in queue."] + + lines = [] + for pack in ps: + lines.append('PACKAGE #%s: "%s" with %d links.' % (pack.pid, pack.name, len(pack.fids) )) + + return lines + + def event_collector(self, args): + ps = self.api.getCollector() + if not ps: + return ["INFO: No packages in collector!"] + + lines = [] + for pack in ps: + lines.append('PACKAGE #%s: "%s" with %d links.' % (pack.pid, pack.name, len(pack.fids) )) + + return lines + + def event_info(self, args): + if not args: + return ['ERROR: Use info like this: info '] + + info = self.api.getFileData(int(args[0])) + + if not info: + return ["ERROR: Link doesn't exists."] + + return ['LINK #%s: %s (%s) [%s][%s]' % (info.fid, info.name, info.format_size, info.status_msg, + info.plugin)] + + def event_packinfo(self, args): + if not args: + return ['ERROR: Use packinfo like this: packinfo '] + + lines = [] + pack = self.api.getPackageData(int(args[0])) + + if not pack: + return ["ERROR: Package doesn't exists."] + + id = args[0] + + self.more = [] + + lines.append('PACKAGE #%s: "%s" with %d links' % (id, pack.name, len(pack.links)) ) + for pyfile in pack.links: + self.more.append('LINK #%s: %s (%s) [%s][%s]' % (pyfile.fid, pyfile.name, pyfile.format_size, + pyfile.statusmsg, pyfile.plugin)) + + if len(self.more) < 6: + lines.extend(self.more) + self.more = [] + else: + lines.extend(self.more[:6]) + self.more = self.more[6:] + lines.append("%d more links do display." % len(self.more)) + + + return lines + + def event_more(self, args): + if not self.more: + return ["No more information to display."] + + lines = self.more[:6] + self.more = self.more[6:] + lines.append("%d more links do display." % len(self.more)) + + return lines + + def event_start(self, args): + + self.api.unpauseServer() + return ["INFO: Starting downloads."] + + def event_stop(self, args): + + self.api.pauseServer() + return ["INFO: No new downloads will be started."] + + + def event_add(self, args): + if len(args) < 2: + return ['ERROR: Add links like this: "add links". ', + 'This will add the link to to the package / the package with id !'] + + + + pack = args[0].strip() + links = [x.strip() for x in args[1:]] + + count_added = 0 + count_failed = 0 + try: + id = int(pack) + pack = self.api.getPackageData(id) + if not pack: + return ["ERROR: Package doesn't exists."] + + #TODO add links + + return ["INFO: Added %d links to Package %s [#%d]" % (len(links), pack["name"], id)] + + except: + # create new package + id = self.api.addPackage(pack, links, 1) + return ["INFO: Created new Package %s [#%d] with %d links." % (pack, id, len(links))] + + + def event_del(self, args): + if len(args) < 2: + return ["ERROR: Use del command like this: del -p|-l [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)"] + + if args[0] == "-p": + ret = self.api.deletePackages(map(int, args[1:])) + return ["INFO: Deleted %d packages!" % len(args[1:])] + + elif args[0] == "-l": + ret = self.api.delLinks(map(int, args[1:])) + return ["INFO: Deleted %d links!" % len(args[1:])] + + else: + return ["ERROR: Use del command like this: del <-p|-l> [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)"] + + def event_push(self, args): + if not args: + return ["ERROR: Push package to queue like this: push "] + + id = int(args[0]) + if not self.api.getPackage_data(id): + return ["ERROR: Package #%d does not exist." % id] + + self.api.pushToQueue(id) + return ["INFO: Pushed package #%d to queue." % id] + + def event_pull(self, args): + if not args: + return ["ERROR: Pull package from queue like this: pull ."] + + id = int(args[0]) + if not self.api.getPackageData(id): + return ["ERROR: Package #%d does not exist." % id] + + self.api.pullFromQueue(id) + return ["INFO: Pulled package #%d from queue to collector." % id] + + def event_c(self, args): + """ captcha answer """ + if not args: + return ["ERROR: Captcha ID missing."] + + task = self.core.captchaManager.getTaskByID(args[0]) + if not task: + return ["ERROR: Captcha Task with ID %s does not exists." % args[0]] + + task.setResult(" ".join(args[1:])) + return ["INFO: Result %s saved." % " ".join(args[1:])] + + + def event_help(self, args): + lines = ["The following commands are available:", + "add [...] Adds link to package. (creates new package if it does not exist)", + "queue Shows all packages in the queue", + "collector Shows all packages in collector", + "del -p|-l [...] Deletes all packages|links with the ids specified", + "info Shows info of the link with id ", + "packinfo Shows info of the package with id ", + "more Shows more info when the result was truncated", + "start Starts all downloads", + "stop Stops the download (but not abort active downloads)", + "push Push package to queue", + "pull Pull package from queue", + "status Show general download status", + "help Shows this help message"] + return lines + + +class IRCError(Exception): + def __init__(self, value): + Exception.__init__(value) + self.value = value + def __str__(self): + return repr(self.value) diff --git a/module/plugins/addons/MergeFiles.py b/module/plugins/addons/MergeFiles.py new file mode 100644 index 000000000..48f997681 --- /dev/null +++ b/module/plugins/addons/MergeFiles.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: and9000 +""" + +import os +import re +import sys +import traceback + +from os.path import join +from module.utils import save_join, fs_encode +from module.plugins.Addon import Addon + +BUFFER_SIZE = 4096 + +class MergeFiles(Addon): + __name__ = "MergeFiles" + __version__ = "0.1" + __description__ = "Merges parts splitted with hjsplit" + __config__ = [ + ("activated" , "bool" , "Activated" , "False"), + ] + __threaded__ = ["packageFinished"] + __author_name__ = ("and9000") + __author_mail__ = ("me@has-no-mail.com") + + def setup(self): + # nothing to do + pass + + def packageFinished(self, pack): + files = {} + fid_dict = {} + for fid, data in pack.getChildren().iteritems(): + if re.search("\.[0-9]{3}$", data["name"]): + if data["name"][:-4] not in files: + files[data["name"][:-4]] = [] + files[data["name"][:-4]].append(data["name"]) + files[data["name"][:-4]].sort() + fid_dict[data["name"]] = fid + + download_folder = self.core.config['general']['download_folder'] + + if self.core.config['general']['folder_per_package']: + download_folder = save_join(download_folder, pack.folder) + + for name, file_list in files.iteritems(): + self.core.log.info("Starting merging of %s" % name) + final_file = open(join(download_folder, fs_encode(name)), "wb") + + for splitted_file in file_list: + self.core.log.debug("Merging part %s" % splitted_file) + pyfile = self.core.files.getFile(fid_dict[splitted_file]) + pyfile.setStatus("processing") + try: + s_file = open(os.path.join(download_folder, splitted_file), "rb") + size_written = 0 + s_file_size = int(os.path.getsize(os.path.join(download_folder, splitted_file))) + while True: + f_buffer = s_file.read(BUFFER_SIZE) + if f_buffer: + final_file.write(f_buffer) + size_written += BUFFER_SIZE + pyfile.setProgress((size_written*100)/s_file_size) + else: + break + s_file.close() + self.core.log.debug("Finished merging part %s" % splitted_file) + except Exception, e: + print traceback.print_exc() + finally: + pyfile.setProgress(100) + pyfile.setStatus("finished") + pyfile.release() + + final_file.close() + self.core.log.info("Finished merging of %s" % name) + + diff --git a/module/plugins/addons/MultiHome.py b/module/plugins/addons/MultiHome.py new file mode 100644 index 000000000..af3f55416 --- /dev/null +++ b/module/plugins/addons/MultiHome.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay +""" + +from module.plugins.Addon import Addon +from time import time + +class MultiHome(Addon): + __name__ = "MultiHome" + __version__ = "0.1" + __description__ = """ip address changer""" + __config__ = [ ("activated", "bool", "Activated" , "False"), + ("interfaces", "str", "Interfaces" , "None") ] + __author_name__ = ("mkaay") + __author_mail__ = ("mkaay@mkaay.de") + + def setup(self): + self.register = {} + self.interfaces = [] + self.parseInterfaces(self.getConfig("interfaces").split(";")) + if not self.interfaces: + self.parseInterfaces([self.config["download"]["interface"]]) + self.setConfig("interfaces", self.toConfig()) + + def toConfig(self): + return ";".join([i.adress for i in self.interfaces]) + + def parseInterfaces(self, interfaces): + for interface in interfaces: + if not interface or str(interface).lower() == "none": + continue + self.interfaces.append(Interface(interface)) + + def coreReady(self): + requestFactory = self.core.requestFactory + oldGetRequest = requestFactory.getRequest + def getRequest(pluginName, account=None): + iface = self.bestInterface(pluginName, account) + if iface: + iface.useFor(pluginName, account) + requestFactory.iface = lambda: iface.adress + self.log.debug("Multihome: using address: "+iface.adress) + return oldGetRequest(pluginName, account) + requestFactory.getRequest = getRequest + + def bestInterface(self, pluginName, account): + best = None + for interface in self.interfaces: + if not best or interface.lastPluginAccess(pluginName, account) < best.lastPluginAccess(pluginName, account): + best = interface + return best + +class Interface(object): + def __init__(self, adress): + self.adress = adress + self.history = {} + + def lastPluginAccess(self, pluginName, account): + if (pluginName, account) in self.history: + return self.history[(pluginName, account)] + return 0 + + def useFor(self, pluginName, account): + self.history[(pluginName, account)] = time() + + def __repr__(self): + return "" % self.adress diff --git a/module/plugins/addons/MultiHoster.py b/module/plugins/addons/MultiHoster.py new file mode 100644 index 000000000..05d25b958 --- /dev/null +++ b/module/plugins/addons/MultiHoster.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from types import MethodType + +from module.plugins.MultiHoster import MultiHoster as MultiHosterAccount, normalize +from module.plugins.Addon import Addon, AddEventListener +from module.plugins.PluginManager import PluginTuple + +class MultiHoster(Addon): + __version__ = "0.1" + __internal__ = True + __description__ = "Gives ability to use MultiHoster services. You need to add your account first." + __config__ = [] + __author_mail__ = ("pyLoad Team",) + __author_mail__ = ("support@pyload.org",) + + #TODO: multiple accounts - multihoster / config options + + def init(self): + + # overwritten plugins + self.plugins = {} + + def addHoster(self, account): + + self.logDebug("New MultiHoster %s" % account.__name__) + + pluginMap = {} + for name in self.core.pluginManager.getPlugins("hoster").keys(): + pluginMap[name.lower()] = name + + supported = [] + new_supported = [] + + for hoster in account.getHosterList(): + name = normalize(hoster) + + if name in pluginMap: + supported.append(pluginMap[name]) + else: + new_supported.append(hoster) + + if not supported and not new_supported: + account.logError(_("No Hoster loaded")) + return + + klass = self.core.pluginManager.getPluginClass(account.__name__) + + # inject plugin plugin + account.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(supported))) + for hoster in supported: + self.plugins[hoster] = klass + + account.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) + + # create new regexp + regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) + + # recreate plugin tuple for new regexp + hoster = self.core.pluginManager.getPlugins("hoster") + p = hoster[account.__name__] + new = PluginTuple(p.version, re.compile(regexp), p.deps, p.user, p.path) + hoster[account.__name__] = new + + + + @AddEventListener("accountDeleted") + def refreshAccounts(self, plugin=None, user=None): + + self.plugins = {} + + for name, account in self.core.accountManager.iterAccounts(): + if isinstance(account, MultiHosterAccount) and account.isUsable(): + self.addHoster(account) + + @AddEventListener("accountUpdated") + def refreshAccount(self, plugin, user): + + account = self.core.accountManager.getAccount(plugin, user) + if isinstance(account, MultiHosterAccount) and account.isUsable(): + self.addHoster(account) + + def activate(self): + self.refreshAccounts() + + # new method for plugin manager + def getPlugin(self2, name): + if name in self.plugins: + return self.plugins[name] + return self2.getPluginClass(name) + + pm = self.core.pluginManager + pm.getPlugin = MethodType(getPlugin, pm, object) + + + def deactivate(self): + #restore state + pm = self.core.pluginManager + pm.getPlugin = pm.getPluginClass + diff --git a/module/plugins/addons/MultishareCz.py b/module/plugins/addons/MultishareCz.py new file mode 100644 index 000000000..a934f43ef --- /dev/null +++ b/module/plugins/addons/MultishareCz.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +from module.network.RequestFactory import getURL +from module.plugins.internal.MultiHoster import MultiHoster +import re + +def getConfigSet(option): + s = set(option.lower().split('|')) + s.discard(u'') + return s + +class MultishareCz(MultiHoster): + __name__ = "MultishareCz" + __version__ = "0.01" + __type__ = "hook" + __config__ = [("activated", "bool", "Activated", "False"), + ("includeHoster", "str", "Use only for downloads from (bar-separated hosters)", ""), + ("excludeHoster", "str", "Do not use for downloads from (bar-separated hosters)", "rapidshare.com|uloz.to")] + __description__ = """MultiShare.cz hook plugin""" + __author_name__ = ("zoidberg") + __author_mail__ = ("zoidberg@mujmail.cz") + + #replacements = [("freakshare.net", "freakshare.com")] + HOSTER_PATTERN = r']*alt="([^"]+)">\s*OK' + + def getHoster(self): + + page = getURL("http://www.multishare.cz/monitoring/") + hoster = set(m.group(1).lower() for m in re.finditer(self.HOSTER_PATTERN, page)) + + option = self.getConfig('includeHoster').strip() + if option: hoster &= getConfigSet(option) + option = self.getConfig('excludeHoster').strip() + if option: hoster -= getConfigSet(option) + + return list(hoster) \ No newline at end of file diff --git a/module/plugins/addons/Premium4Me.py b/module/plugins/addons/Premium4Me.py new file mode 100644 index 000000000..fc3ce2343 --- /dev/null +++ b/module/plugins/addons/Premium4Me.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +from module.network.RequestFactory import getURL +from module.plugins.internal.MultiHoster import MultiHoster + +class Premium4Me(MultiHoster): + __name__ = "Premium4Me" + __version__ = "0.02" + __type__ = "hook" + + __config__ = [("activated", "bool", "Activated", "False"), + ("hosterListMode", "all;listed;unlisted", "Use for downloads from supported hosters:", "all"), + ("hosterList", "str", "Hoster list (comma separated)", "")] + __description__ = """premium4.me hook plugin""" + __author_name__ = ("RaNaN", "zoidberg") + __author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz") + + replacements = [("freakshare.net", "freakshare.com")] + + def getHoster(self): + + page = getURL("http://premium4.me/api/hosters.php?authcode=%s" % self.account.authcode) + hosters = set([x.strip() for x in page.replace("\"", "").split(";")]) + + configMode = self.getConfig('hosterListMode') + if configMode in ("listed", "unlisted"): + configList = set(self.getConfig('hosterList').strip().lower().replace(',','|').split('|')) + configList.discard(u'') + if configMode == "listed": + hosters &= configList + elif configMode == "unlisted": + hosters -= configList + + return list(hosters) + + def coreReady(self): + + self.account = self.core.accountManager.getAccountPlugin("Premium4Me") + + user = self.account.selectAccount()[0] + + if not user: + self.logError(_("Please add your premium4.me account first and restart pyLoad")) + return + + return MultiHoster.coreReady(self) \ No newline at end of file diff --git a/module/plugins/addons/RehostTo.py b/module/plugins/addons/RehostTo.py new file mode 100644 index 000000000..b16987f5c --- /dev/null +++ b/module/plugins/addons/RehostTo.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +from module.network.RequestFactory import getURL +from module.plugins.internal.MultiHoster import MultiHoster + +class RehostTo(MultiHoster): + __name__ = "RehostTo" + __version__ = "0.41" + __type__ = "hook" + + __config__ = [("activated", "bool", "Activated", "False")] + + __description__ = """rehost.to hook plugin""" + __author_name__ = ("RaNaN") + __author_mail__ = ("RaNaN@pyload.org") + + replacements = [("freakshare.net", "freakshare.com")] + + def getHoster(self): + + page = getURL("http://rehost.to/api.php?cmd=get_supported_och_dl&long_ses=%s" % self.long_ses) + return [x.strip() for x in page.replace("\"", "").split(",")] + + + def coreReady(self): + + self.account = self.core.accountManager.getAccountPlugin("RehostTo") + + user = self.account.selectAccount()[0] + + if not user: + self.log.error("Rehost.to: "+ _("Please add your rehost.to account first and restart pyLoad")) + return + + data = self.account.getAccountInfo(user) + self.ses = data["ses"] + self.long_ses = data["long_ses"] + + return MultiHoster.coreReady(self) diff --git a/module/plugins/addons/UpdateManager.py b/module/plugins/addons/UpdateManager.py new file mode 100644 index 000000000..5bc6ac447 --- /dev/null +++ b/module/plugins/addons/UpdateManager.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +import sys +import re +from os import stat +from os.path import join, exists +from time import time + +from module.plugins.PluginManager import IGNORE +from module.network.RequestFactory import getURL +from module.plugins.Addon import threaded, Expose, Addon + +class UpdateManager(Addon): + __name__ = "UpdateManager" + __version__ = "0.12" + __description__ = """checks for updates""" + __config__ = [("activated", "bool", "Activated", "True"), + ("interval", "int", "Check interval in minutes", "360"), + ("debug", "bool", "Check for plugin changes when in debug mode", False)] + __author_name__ = ("RaNaN") + __author_mail__ = ("ranan@pyload.org") + + @property + def debug(self): + return self.core.debug and self.getConfig("debug") + + + def setup(self): + if self.debug: + self.logDebug("Monitoring file changes") + self.interval = 4 + self.last_check = 0 #timestamp of updatecheck + self.old_periodical = self.periodical + self.periodical = self.checkChanges + self.mtimes = {} #recordes times + else: + self.interval = self.getConfig("interval") * 60 + + self.updated = False + self.reloaded = True + + self.info = {"pyload": False, "plugins": False} + + @threaded + def periodical(self): + + if self.core.version.endswith("-dev"): + self.logDebug("No update check performed on dev version.") + return + + update = self.checkForUpdate() + if update: + self.info["pyload"] = True + else: + self.log.info(_("No Updates for pyLoad")) + self.checkPlugins() + + if self.updated and not self.reloaded: + self.info["plugins"] = True + self.log.info(_("*** Plugins have been updated, please restart pyLoad ***")) + elif self.updated and self.reloaded: + self.log.info(_("Plugins updated and reloaded")) + self.updated = False + else: + self.log.info(_("No plugin updates available")) + + @Expose + def recheckForUpdates(self): + """recheck if updates are available""" + self.periodical() + + def checkForUpdate(self): + """checks if an update is available""" + + try: + version_check = getURL("http://get.pyload.org/check/%s/" % self.core.api.getServerVersion()) + if version_check == "": + return False + else: + self.log.info(_("*** New pyLoad Version %s available ***") % version_check) + self.log.info(_("*** Get it here: http://pyload.org/download ***")) + return True + except: + self.log.warning(_("Not able to connect server for updates")) + return False + + + def checkPlugins(self): + """ checks for plugins updates""" + + # plugins were already updated + if self.info["plugins"]: return + + try: + updates = getURL("http://get.pyload.org/plugins/check/") + except: + self.log.warning(_("Not able to connect server for updates")) + return False + + updates = updates.splitlines() + reloads = [] + + vre = re.compile(r'__version__.*=.*("|\')([0-9.]+)') + + for plugin in updates: + path, version = plugin.split(":") + prefix, filename = path.split("/") + + if filename.endswith(".pyc"): + name = filename[:filename.find("_")] + else: + name = filename.replace(".py", "") + + #TODO: obsolete + if prefix.endswith("s"): + type = prefix[:-1] + else: + type = prefix + + plugins = self.core.pluginManager.getPlugins(type) + + if name in plugins: + if float(plugins[name].version) >= float(version): + continue + + if name in IGNORE or (type, name) in IGNORE: + continue + + self.log.info(_("New version of %(type)s|%(name)s : %(version).2f") % { + "type": type, + "name": name, + "version": float(version) + }) + + try: + content = getURL("http://get.pyload.org/plugins/get/" + path) + except Exception, e: + self.logWarning(_("Error when updating %s") % filename, str(e)) + continue + + m = vre.search(content) + if not m or m.group(2) != version: + self.logWarning(_("Error when updating %s") % name, _("Version mismatch")) + continue + + f = open(join("userplugins", prefix, filename), "wb") + f.write(content) + f.close() + self.updated = True + + reloads.append((prefix, name)) + + self.reloaded = self.core.pluginManager.reloadPlugins(reloads) + + def checkChanges(self): + + if self.last_check + self.getConfig("interval") * 60 < time(): + self.old_periodical() + self.last_check = time() + + modules = filter( + lambda m: m and (m.__name__.startswith("module.plugins.") or m.__name__.startswith("userplugins.")) and m.__name__.count(".") >= 2, + sys.modules.itervalues()) + + reloads = [] + + for m in modules: + root, type, name = m.__name__.rsplit(".", 2) + id = (type, name) + if type in self.core.pluginManager.plugins: + f = m.__file__.replace(".pyc", ".py") + if not exists(f): continue + + mtime = stat(f).st_mtime + + if id not in self.mtimes: + self.mtimes[id] = mtime + elif self.mtimes[id] < mtime: + reloads.append(id) + self.mtimes[id] = mtime + + self.core.pluginManager.reloadPlugins(reloads) diff --git a/module/plugins/addons/XMPPInterface.py b/module/plugins/addons/XMPPInterface.py new file mode 100644 index 000000000..e8ef1d2ca --- /dev/null +++ b/module/plugins/addons/XMPPInterface.py @@ -0,0 +1,276 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN + @interface-version: 0.2 +""" + +from pyxmpp import streamtls +from pyxmpp.all import JID, Message, Presence +from pyxmpp.jabber.client import JabberClient +from pyxmpp.interface import implements +from pyxmpp.interfaces import * + +from module.plugins.addons.IRCInterface import IRCInterface + +class XMPPInterface(IRCInterface, JabberClient): + __name__ = "XMPPInterface" + __version__ = "0.1" + __description__ = """connect to jabber and let owner perform different tasks""" + __config__ = [("activated", "bool", "Activated", "False"), + ("jid", "str", "Jabber ID", "user@exmaple-jabber-server.org"), + ("pw", "str", "Password", ""), + ("tls", "bool", "Use TLS", False), + ("owners", "str", "List of JIDs accepting commands from", "me@icq-gateway.org;some@msn-gateway.org"), + ("info_file", "bool", "Inform about every file finished", "False"), + ("info_pack", "bool", "Inform about every package finished", "True"), + ("captcha", "bool", "Send captcha requests", "True")] + __author_name__ = ("RaNaN") + __author_mail__ = ("RaNaN@pyload.org") + + implements(IMessageHandlersProvider) + + def __init__(self, core, manager): + IRCInterface.__init__(self, core, manager) + + self.jid = JID(self.getConfig("jid")) + password = self.getConfig("pw") + + # if bare JID is provided add a resource -- it is required + if not self.jid.resource: + self.jid = JID(self.jid.node, self.jid.domain, "pyLoad") + + if self.getConfig("tls"): + tls_settings = streamtls.TLSSettings(require=True, verify_peer=False) + auth = ("sasl:PLAIN", "sasl:DIGEST-MD5") + else: + tls_settings = None + auth = ("sasl:DIGEST-MD5", "digest") + + # setup client with provided connection information + # and identity data + JabberClient.__init__(self, self.jid, password, + disco_name="pyLoad XMPP Client", disco_type="bot", + tls_settings=tls_settings, auth_methods=auth) + + self.interface_providers = [ + VersionHandler(self), + self, + ] + + def coreReady(self): + self.new_package = {} + + self.start() + + def packageFinished(self, pypack): + try: + if self.getConfig("info_pack"): + self.announce(_("Package finished: %s") % pypack.name) + except: + pass + + def downloadFinished(self, pyfile): + try: + if self.getConfig("info_file"): + self.announce( + _("Download finished: %(name)s @ %(plugin)s") % {"name": pyfile.name, "plugin": pyfile.pluginname}) + except: + pass + + def run(self): + # connect to IRC etc. + self.connect() + try: + self.loop() + except Exception, ex: + self.core.log.error("pyLoad XMPP: %s" % str(ex)) + + def stream_state_changed(self, state, arg): + """This one is called when the state of stream connecting the component + to a server changes. This will usually be used to let the user + know what is going on.""" + self.log.debug("pyLoad XMPP: *** State changed: %s %r ***" % (state, arg)) + + def disconnected(self): + self.log.debug("pyLoad XMPP: Client was disconnected") + + def stream_closed(self, stream): + self.log.debug("pyLoad XMPP: Stream was closed | %s" % stream) + + def stream_error(self, err): + self.log.debug("pyLoad XMPP: Stream Error: %s" % err) + + def get_message_handlers(self): + """Return list of (message_type, message_handler) tuples. + + The handlers returned will be called when matching message is received + in a client session.""" + return [ + ("normal", self.message), + ] + + def presence_control(self, stanza): + from_jid = unicode(stanza.get_from_jid()) + stanza_type = stanza.get_type() + self.log.debug("pyLoad XMPP: %s stanza from %s" % (stanza_type, + from_jid)) + + if from_jid in self.getConfig("owners"): + return stanza.make_accept_response() + + return stanza.make_deny_response() + + def session_started(self): + self.stream.send(Presence()) + + self.stream.set_presence_handler("subscribe", self.presence_control) + self.stream.set_presence_handler("subscribed", self.presence_control) + self.stream.set_presence_handler("unsubscribe", self.presence_control) + self.stream.set_presence_handler("unsubscribed", self.presence_control) + + def message(self, stanza): + """Message handler for the component.""" + subject = stanza.get_subject() + body = stanza.get_body() + t = stanza.get_type() + self.log.debug(u'pyLoad XMPP: Message from %s received.' % (unicode(stanza.get_from(), ))) + self.log.debug(u'pyLoad XMPP: Body: %s Subject: %s Type: %s' % (body, subject, t)) + + if t == "headline": + # 'headline' messages should never be replied to + return True + if subject: + subject = u"Re: " + subject + + to_jid = stanza.get_from() + from_jid = stanza.get_to() + + #j = JID() + to_name = to_jid.as_utf8() + from_name = from_jid.as_utf8() + + names = self.getConfig("owners").split(";") + + if to_name in names or to_jid.node + "@" + to_jid.domain in names: + messages = [] + + trigger = "pass" + args = None + + try: + temp = body.split() + trigger = temp[0] + if len(temp) > 1: + args = temp[1:] + except: + pass + + handler = getattr(self, "event_%s" % trigger, self.event_pass) + try: + res = handler(args) + for line in res: + m = Message( + to_jid=to_jid, + from_jid=from_jid, + stanza_type=stanza.get_type(), + subject=subject, + body=line) + + messages.append(m) + except Exception, e: + self.log.error("pyLoad XMPP: " + repr(e)) + + return messages + + else: + return True + + def response(self, msg, origin=""): + return self.announce(msg) + + def announce(self, message): + """ send message to all owners""" + for user in self.getConfig("owners").split(";"): + self.log.debug("pyLoad XMPP: Send message to %s" % user) + + to_jid = JID(user) + + m = Message(from_jid=self.jid, + to_jid=to_jid, + stanza_type="chat", + body=message) + + stream = self.get_stream() + if not stream: + self.connect() + stream = self.get_stream() + + stream.send(m) + + def beforeReconnecting(self, ip): + self.disconnect() + + def afterReconnecting(self, ip): + self.connect() + + +class VersionHandler(object): + """Provides handler for a version query. + + This class will answer version query and announce 'jabber:iq:version' namespace + in the client's disco#info results.""" + + implements(IIqHandlersProvider, IFeaturesProvider) + + def __init__(self, client): + """Just remember who created this.""" + self.client = client + + def get_features(self): + """Return namespace which should the client include in its reply to a + disco#info query.""" + return ["jabber:iq:version"] + + def get_iq_get_handlers(self): + """Return list of tuples (element_name, namespace, handler) describing + handlers of stanzas""" + return [ + ("query", "jabber:iq:version", self.get_version), + ] + + def get_iq_set_handlers(self): + """Return empty list, as this class provides no stanza handler.""" + return [] + + def get_version(self, iq): + """Handler for jabber:iq:version queries. + + jabber:iq:version queries are not supported directly by PyXMPP, so the + XML node is accessed directly through the libxml2 API. This should be + used very carefully!""" + iq = iq.make_result_response() + q = iq.new_query("jabber:iq:version") + q.newTextChild(q.ns(), "name", "Echo component") + q.newTextChild(q.ns(), "version", "1.0") + return iq + + def unload(self): + self.log.debug("pyLoad XMPP: unloading") + self.disconnect() + + def deactivate(self): + self.unload() diff --git a/module/plugins/addons/__init__.py b/module/plugins/addons/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/module/plugins/hooks/CaptchaTrader.py b/module/plugins/hooks/CaptchaTrader.py deleted file mode 100644 index 889eb83c0..000000000 --- a/module/plugins/hooks/CaptchaTrader.py +++ /dev/null @@ -1,159 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay, RaNaN -""" - -try: - from json import loads -except ImportError: - from simplejson import loads - -from thread import start_new_thread -from pycurl import FORM_FILE, LOW_SPEED_TIME - -from module.network.RequestFactory import getURL, getRequest -from module.network.HTTPRequest import BadHeader - -from module.plugins.Hook import Hook - -PYLOAD_KEY = "9f65e7f381c3af2b076ea680ae96b0b7" - -class CaptchaTraderException(Exception): - def __init__(self, err): - self.err = err - - def getCode(self): - return self.err - - def __str__(self): - return "" % self.err - - def __repr__(self): - return "" % self.err - -class CaptchaTrader(Hook): - __name__ = "CaptchaTrader" - __version__ = "0.13" - __description__ = """send captchas to captchatrader.com""" - __config__ = [("activated", "bool", "Activated", True), - ("username", "str", "Username", ""), - ("force", "bool", "Force CT even if client is connected", False), - ("passkey", "password", "Password", ""),] - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - SUBMIT_URL = "http://captchatrader.com/api/submit" - RESPOND_URL = "http://captchatrader.com/api/respond" - GETCREDITS_URL = "http://captchatrader.com/api/get_credits/username:%(user)s/password:%(password)s/" - - def setup(self): - self.info = {} - - def getCredits(self): - json = getURL(CaptchaTrader.GETCREDITS_URL % {"user": self.getConfig("username"), - "password": self.getConfig("passkey")}) - response = loads(json) - if response[0] < 0: - raise CaptchaTraderException(response[1]) - else: - self.logInfo(_("%s credits left") % response[1]) - self.info["credits"] = response[1] - return response[1] - - def submit(self, captcha, captchaType="file", match=None): - if not PYLOAD_KEY: - raise CaptchaTraderException("No API Key Specified!") - - #if type(captcha) == str and captchaType == "file": - # raise CaptchaTraderException("Invalid Type") - assert captchaType in ("file", "url-jpg", "url-jpeg", "url-png", "url-bmp") - - req = getRequest() - - #raise timeout threshold - req.c.setopt(LOW_SPEED_TIME, 80) - - try: - json = req.load(CaptchaTrader.SUBMIT_URL, post={"api_key": PYLOAD_KEY, - "username": self.getConfig("username"), - "password": self.getConfig("passkey"), - "value": (FORM_FILE, captcha), - "type": captchaType}, multipart=True) - finally: - req.close() - - response = loads(json) - if response[0] < 0: - raise CaptchaTraderException(response[1]) - - ticket = response[0] - result = response[1] - self.logDebug("result %s : %s" % (ticket,result)) - - return ticket, result - - def respond(self, ticket, success): - try: - json = getURL(CaptchaTrader.RESPOND_URL, post={"is_correct": 1 if success else 0, - "username": self.getConfig("username"), - "password": self.getConfig("passkey"), - "ticket": ticket}) - - response = loads(json) - if response[0] < 0: - raise CaptchaTraderException(response[1]) - - except BadHeader, e: - self.logError(_("Could not send response."), str(e)) - - def newCaptchaTask(self, task): - if not task.isTextual(): - return False - - if not self.getConfig("username") or not self.getConfig("passkey"): - return False - - if self.core.isClientConnected() and not self.getConfig("force"): - return False - - if self.getCredits() > 10: - task.handler.append(self) - task.setWaiting(100) - start_new_thread(self.processCaptcha, (task,)) - - else: - self.logInfo(_("Your CaptchaTrader Account has not enough credits")) - - def captchaCorrect(self, task): - if "ticket" in task.data: - ticket = task.data["ticket"] - self.respond(ticket, True) - - def captchaInvalid(self, task): - if "ticket" in task.data: - ticket = task.data["ticket"] - self.respond(ticket, False) - - def processCaptcha(self, task): - c = task.captchaFile - try: - ticket, result = self.submit(c) - except CaptchaTraderException, e: - task.error = e.getCode() - return - - task.data["ticket"] = ticket - task.setResult(result) diff --git a/module/plugins/hooks/ClickAndLoad.py b/module/plugins/hooks/ClickAndLoad.py deleted file mode 100644 index fc32d0da8..000000000 --- a/module/plugins/hooks/ClickAndLoad.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN - @interface-version: 0.2 -""" - -import socket -import thread - -from module.plugins.Hook import Hook - -class ClickAndLoad(Hook): - __name__ = "ClickAndLoad" - __version__ = "0.2" - __description__ = """Gives abillity to use jd's click and load. depends on webinterface""" - __config__ = [("activated", "bool", "Activated", "True"), - ("extern", "bool", "Allow external link adding", "False")] - __author_name__ = ("RaNaN", "mkaay") - __author_mail__ = ("RaNaN@pyload.de", "mkaay@mkaay.de") - - def activate(self): - self.port = int(self.core.config['webinterface']['port']) - if self.core.config['webinterface']['activated']: - try: - if self.getConfig("extern"): - ip = "0.0.0.0" - else: - ip = "127.0.0.1" - - thread.start_new_thread(proxy, (self, ip, self.port, 9666)) - except: - self.log.error("ClickAndLoad port already in use.") - - -def proxy(self, *settings): - thread.start_new_thread(server, (self,) + settings) - lock = thread.allocate_lock() - lock.acquire() - lock.acquire() - - -def server(self, *settings): - try: - dock_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - dock_socket.bind((settings[0], settings[2])) - dock_socket.listen(5) - while True: - client_socket = dock_socket.accept()[0] - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.connect(("127.0.0.1", settings[1])) - thread.start_new_thread(forward, (client_socket, server_socket)) - thread.start_new_thread(forward, (server_socket, client_socket)) - except socket.error, e: - if hasattr(e, "errno"): - errno = e.errno - else: - errno = e.args[0] - - if errno == 98: - self.core.log.warning(_("Click'N'Load: Port 9666 already in use")) - return - thread.start_new_thread(server, (self,) + settings) - except: - thread.start_new_thread(server, (self,) + settings) - - -def forward(source, destination): - string = ' ' - while string: - string = source.recv(1024) - if string: - destination.sendall(string) - else: - #source.shutdown(socket.SHUT_RD) - destination.shutdown(socket.SHUT_WR) diff --git a/module/plugins/hooks/EasybytezCom.py b/module/plugins/hooks/EasybytezCom.py deleted file mode 100644 index 4dd39cca6..000000000 --- a/module/plugins/hooks/EasybytezCom.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.network.RequestFactory import getURL -from module.plugins.internal.MultiHoster import MultiHoster -import re - -def getConfigSet(option): - s = set(option.lower().replace(',','|').split('|')) - s.discard(u'') - return s - -class EasybytezCom(MultiHoster): - __name__ = "EasybytezCom" - __version__ = "0.01" - __type__ = "hook" - __config__ = [("activated", "bool", "Activated", "False"), - ("includeHoster", "str", "Use only for downloads from (comma-separated hosters)", ""), - ("excludeHoster", "str", "Do not use for downloads from (comma-separated hosters)", "")] - __description__ = """EasyBytez.com hook plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - def getHoster(self): - - hoster = set(['2shared.com', 'easy-share.com', 'filefactory.com', 'fileserve.com', 'filesonic.com', 'hotfile.com', 'mediafire.com', 'megaupload.com', 'netload.in', 'rapidshare.com', 'uploading.com', 'wupload.com', 'oron.com', 'uploadstation.com', 'ul.to', 'uploaded.to']) - - option = self.getConfig('includeHoster').strip() - if option: hoster &= getConfigSet(option) - option = self.getConfig('excludeHoster').strip() - if option: hoster -= getConfigSet(option) - - return list(hoster) \ No newline at end of file diff --git a/module/plugins/hooks/Ev0InFetcher.py b/module/plugins/hooks/Ev0InFetcher.py deleted file mode 100644 index 0cd3f3226..000000000 --- a/module/plugins/hooks/Ev0InFetcher.py +++ /dev/null @@ -1,87 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" -from module.lib import feedparser -from time import mktime, time - -from module.plugins.Hook import Hook - -class Ev0InFetcher(Hook): - __name__ = "Ev0InFetcher" - __version__ = "0.2" - __description__ = """checks rss feeds for ev0.in""" - __config__ = [("activated", "bool", "Activated", "False"), - ("interval", "int", "Check interval in minutes", "10"), - ("queue", "bool", "Move new shows directly to Queue", False), - ("shows", "str", "Shows to check for (comma seperated)", ""), - ("quality", "xvid;x264;rmvb", "Video Format", "xvid"), - ("hoster", "str", "Hoster to use (comma seperated)", "NetloadIn,RapidshareCom,MegauploadCom,HotfileCom")] - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def setup(self): - self.interval = self.getConfig("interval") * 60 - - def filterLinks(self, links): - results = self.core.pluginManager.parseUrls(links) - sortedLinks = {} - - for url, hoster in results[0]: - if hoster not in sortedLinks: - sortedLinks[hoster] = [] - sortedLinks[hoster].append(url) - - for h in self.getConfig("hoster").split(","): - try: - return sortedLinks[h.strip()] - except: - continue - return [] - - def periodical(self): - def normalizefiletitle(filename): - filename = filename.replace('.', ' ') - filename = filename.replace('_', ' ') - filename = filename.lower() - return filename - - shows = [s.strip() for s in self.getConfig("shows").split(",")] - - feed = feedparser.parse("http://feeds.feedburner.com/ev0in/%s?format=xml" % self.getConfig("quality")) - - showStorage = {} - for show in shows: - showStorage[show] = int(self.getStorage("show_%s_lastfound" % show, 0)) - - found = False - for item in feed['items']: - for show, lastfound in showStorage.iteritems(): - if show.lower() in normalizefiletitle(item['title']) and lastfound < int(mktime(item.date_parsed)): - links = self.filterLinks(item['description'].split("
")) - packagename = item['title'].encode("utf-8") - self.core.log.info("Ev0InFetcher: new episode '%s' (matched '%s')" % (packagename, show)) - self.core.api.addPackage(packagename, links, 1 if self.getConfig("queue") else 0) - self.setStorage("show_%s_lastfound" % show, int(mktime(item.date_parsed))) - found = True - if not found: - #self.core.log.debug("Ev0InFetcher: no new episodes found") - pass - - for show, lastfound in self.getStorage().iteritems(): - if int(lastfound) > 0 and int(lastfound) + (3600*24*30) < int(time()): - self.delStorage("show_%s_lastfound" % show) - self.core.log.debug("Ev0InFetcher: cleaned '%s' record" % show) diff --git a/module/plugins/hooks/ExternalScripts.py b/module/plugins/hooks/ExternalScripts.py deleted file mode 100644 index 39fe2b9f0..000000000 --- a/module/plugins/hooks/ExternalScripts.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -import subprocess -from os import access, X_OK, makedirs -from os.path import basename - -from module.plugins.Hook import Hook -from module.utils.fs import save_join, exists, join, listdir - -class ExternalScripts(Hook): - __name__ = "ExternalScripts" - __version__ = "0.21" - __description__ = """Run external scripts""" - __config__ = [("activated", "bool", "Activated", "True")] - __author_name__ = ("mkaay", "RaNaN", "spoob") - __author_mail__ = ("mkaay@mkaay.de", "ranan@pyload.org", "spoob@pyload.org") - - event_list = ["unrarFinished", "allDownloadsFinished", "allDownloadsProcessed"] - - def setup(self): - self.scripts = {} - - folders = ['download_preparing', 'download_finished', 'package_finished', - 'before_reconnect', 'after_reconnect', 'unrar_finished', - 'all_dls_finished', 'all_dls_processed'] - - for folder in folders: - - self.scripts[folder] = [] - - self.initPluginType(folder, join(pypath, 'scripts', folder)) - self.initPluginType(folder, join('scripts', folder)) - - for script_type, names in self.scripts.iteritems(): - if names: - self.logInfo((_("Installed scripts for %s: ") % script_type ) + ", ".join([basename(x) for x in names])) - - - def initPluginType(self, folder, path): - if not exists(path): - try: - makedirs(path) - except : - self.logDebug("Script folder %s not created" % folder) - return - - for f in listdir(path): - if f.startswith("#") or f.startswith(".") or f.startswith("_") or f.endswith("~") or f.endswith(".swp"): - continue - - if not access(join(path,f), X_OK): - self.logWarning(_("Script not executable:") + " %s/%s" % (folder, f)) - - self.scripts[folder].append(join(path, f)) - - def callScript(self, script, *args): - try: - cmd = [script] + [str(x) if not isinstance(x, basestring) else x for x in args] - #output goes to pyload - subprocess.Popen(cmd, bufsize=-1) - except Exception, e: - self.logError(_("Error in %(script)s: %(error)s") % { "script" :basename(script), "error": str(e)}) - - def downloadPreparing(self, pyfile): - for script in self.scripts['download_preparing']: - self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.id) - - def downloadFinished(self, pyfile): - for script in self.scripts['download_finished']: - self.callScript(script, pyfile.pluginname, pyfile.url, pyfile.name, pyfile.id, - save_join(self.core.config['general']['download_folder'], pyfile.package().folder, pyfile.name), - pyfile.id) - - - def packageFinished(self, pypack): - for script in self.scripts['package_finished']: - folder = self.core.config['general']['download_folder'] - folder = save_join(folder, pypack.folder) - - self.callScript(script, pypack.name, folder, pypack.id) - - def beforeReconnecting(self, ip): - for script in self.scripts['before_reconnect']: - self.callScript(script, ip) - - def afterReconnecting(self, ip): - for script in self.scripts['after_reconnect']: - self.callScript(script, ip) - - def unrarFinished(self, folder, fname): - for script in self.scripts["unrar_finished"]: - self.callScript(script, folder, fname) - - def allDownloadsFinished(self): - for script in self.scripts["all_dls_finished"]: - self.callScript(script) - - def allDownloadsProcessed(self): - for script in self.scripts["all_dls_processed"]: - self.callScript(script) - diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py deleted file mode 100644 index 12bd40d1b..000000000 --- a/module/plugins/hooks/ExtractArchive.py +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import sys -import os -from os.path import basename, isfile, isdir, join -from traceback import print_exc -from copy import copy - -# monkey patch bug in python 2.6 and lower -# see http://bugs.python.org/issue6122 -# http://bugs.python.org/issue1236 -# http://bugs.python.org/issue1731717 -if sys.version_info < (2, 7) and os.name != "nt": - from subprocess import Popen - import errno - - def _eintr_retry_call(func, *args): - while True: - try: - return func(*args) - except OSError, e: - if e.errno == errno.EINTR: - continue - raise - - # unsued timeout option for older python version - def wait(self, timeout=0): - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode is None: - try: - pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) - except OSError, e: - if e.errno != errno.ECHILD: - raise - # This happens if SIGCLD is set to be ignored or waiting - # for child processes has otherwise been disabled for our - # process. This child is dead, we can't get the status. - sts = 0 - self._handle_exitstatus(sts) - return self.returncode - - Popen.wait = wait - -if os.name != "nt": - from os import chown - from pwd import getpwnam - from grp import getgrnam - -from module.utils.fs import save_join, fs_encode, exists, remove, chmod, makedirs -from module.plugins.Hook import Hook, threaded, Expose -from module.plugins.internal.AbstractExtractor import ArchiveError, CRCError, WrongPassword - -class ExtractArchive(Hook): - """ - Provides: unrarFinished (folder, filename) - """ - __name__ = "ExtractArchive" - __version__ = "0.12" - __description__ = "Extract different kind of archives" - __config__ = [("activated", "bool", "Activated", True), - ("fullpath", "bool", "Extract full path", True), - ("overwrite", "bool", "Overwrite files", True), - ("passwordfile", "file", "password file", "unrar_passwords.txt"), - ("deletearchive", "bool", "Delete archives when done", False), - ("subfolder", "bool", "Create subfolder for each package", False), - ("destination", "folder", "Extract files to", ""), - ("recursive", "bool", "Extract archives in archvies", True), - ("queue", "bool", "Wait for all downloads to be finished", True), - ("renice", "int", "CPU Priority", 0), ] - __author_name__ = ("pyload Team") - __author_mail__ = ("adminpyload.org") - - event_list = ["allDownloadsProcessed"] - - def setup(self): - self.plugins = [] - self.passwords = [] - names = [] - - for p in ("UnRar", "UnZip"): - try: - module = self.core.pluginManager.loadModule("internal", p) - klass = getattr(module, p) - if klass.checkDeps(): - names.append(p) - self.plugins.append(klass) - - except OSError, e: - if e.errno == 2: - self.logInfo(_("No %s installed") % p) - else: - self.logWarning(_("Could not activate %s") % p, str(e)) - if self.core.debug: - print_exc() - - except Exception, e: - self.logWarning(_("Could not activate %s") % p, str(e)) - if self.core.debug: - print_exc() - - if names: - self.logInfo(_("Activated") + " " + " ".join(names)) - else: - self.logInfo(_("No Extract plugins activated")) - - # queue with package ids - self.queue = [] - - @Expose - def extractPackage(self, id): - """ Extract package with given id""" - self.manager.startThread(self.extract, [id]) - - def packageFinished(self, pypack): - if self.getConfig("queue"): - self.logInfo(_("Package %s queued for later extracting") % pypack.name) - self.queue.append(pypack.id) - else: - self.manager.startThread(self.extract, [pypack.id]) - - - @threaded - def allDownloadsProcessed(self, thread): - local = copy(self.queue) - del self.queue[:] - self.extract(local, thread) - - - def extract(self, ids, thread=None): - # reload from txt file - self.reloadPasswords() - - # dl folder - dl = self.config['general']['download_folder'] - - extracted = [] - - #iterate packages -> plugins -> targets - for pid in ids: - p = self.core.files.getPackage(pid) - self.logInfo(_("Check package %s") % p.name) - if not p: continue - - # determine output folder - out = save_join(dl, p.folder, "") - # force trailing slash - - if self.getConfig("destination") and self.getConfig("destination").lower() != "none": - - out = save_join(dl, p.folder, self.getConfig("destination"), "") - #relative to package folder if destination is relative, otherwise absolute path overwrites them - - if self.getConf("subfolder"): - out = join(out, fs_encode(p.folder)) - - if not exists(out): - makedirs(out) - - files_ids = [(save_join(dl, p.folder, x["name"]), x["id"]) for x in p.getChildren().itervalues()] - matched = False - - # check as long there are unseen files - while files_ids: - new_files_ids = [] - - for plugin in self.plugins: - targets = plugin.getTargets(files_ids) - if targets: - self.logDebug("Targets for %s: %s" % (plugin.__name__, targets)) - matched = True - for target, fid in targets: - if target in extracted: - self.logDebug(basename(target), "skipped") - continue - extracted.append(target) #prevent extracting same file twice - - klass = plugin(self, target, out, self.getConfig("fullpath"), self.getConfig("overwrite"), - self.getConfig("renice")) - klass.init() - - self.logInfo(basename(target), _("Extract to %s") % out) - new_files = self.startExtracting(klass, fid, p.password.strip().splitlines(), thread) - self.logDebug("Extracted: %s" % new_files) - self.setPermissions(new_files) - - for file in new_files: - if not exists(file): - self.logDebug("new file %s does not exists" % file) - continue - if self.getConfig("recursive") and isfile(file): - new_files_ids.append((file, fid)) #append as new target - - files_ids = new_files_ids # also check extracted files - - if not matched: self.logInfo(_("No files found to extract")) - - - - def startExtracting(self, plugin, fid, passwords, thread): - pyfile = self.core.files.getFile(fid) - if not pyfile: return [] - - pyfile.setCustomStatus(_("extracting")) - thread.addActive(pyfile) #keep this file until everything is done - - try: - progress = lambda x: pyfile.setProgress(x) - success = False - - if not plugin.checkArchive(): - plugin.extract(progress) - success = True - else: - self.logInfo(basename(plugin.file), _("Password protected")) - self.logDebug("Passwords: %s" % str(passwords)) - - pwlist = copy(self.getPasswords()) - #remove already supplied pws from list (only local) - for pw in passwords: - if pw in pwlist: pwlist.remove(pw) - - for pw in passwords + pwlist: - try: - self.logDebug("Try password: %s" % pw) - if plugin.checkPassword(pw): - plugin.extract(progress, pw) - self.addPassword(pw) - success = True - break - except WrongPassword: - self.logDebug("Password was wrong") - - if not success: - self.logError(basename(plugin.file), _("Wrong password")) - return [] - - if self.core.debug: - self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles())) - - if self.getConfig("deletearchive"): - files = plugin.getDeleteFiles() - self.logInfo(_("Deleting %s files") % len(files)) - for f in files: - if exists(f): remove(f) - else: self.logDebug("%s does not exists" % f) - - self.logInfo(basename(plugin.file), _("Extracting finished")) - self.manager.dispatchEvent("unrarFinished", plugin.out, plugin.file) - - return plugin.getExtractedFiles() - - - except ArchiveError, e: - self.logError(basename(plugin.file), _("Archive Error"), str(e)) - except CRCError: - self.logError(basename(plugin.file), _("CRC Mismatch")) - except Exception, e: - if self.core.debug: - print_exc() - self.logError(basename(plugin.file), _("Unknown Error"), str(e)) - - return [] - - @Expose - def getPasswords(self): - """ List of saved passwords """ - return self.passwords - - - def reloadPasswords(self): - pwfile = self.getConfig("passwordfile") - if not exists(pwfile): - open(pwfile, "wb").close() - - passwords = [] - f = open(pwfile, "rb") - for pw in f.read().splitlines(): - passwords.append(pw) - f.close() - - self.passwords = passwords - - - @Expose - def addPassword(self, pw): - """ Adds a password to saved list""" - pwfile = self.getConfig("passwordfile") - - if pw in self.passwords: self.passwords.remove(pw) - self.passwords.insert(0, pw) - - f = open(pwfile, "wb") - for pw in self.passwords: - f.write(pw + "\n") - f.close() - - def setPermissions(self, files): - for f in files: - if not exists(f): continue - try: - if self.core.config["permission"]["change_file"]: - if isfile(f): - chmod(f, int(self.core.config["permission"]["file"], 8)) - elif isdir(f): - chmod(f, int(self.core.config["permission"]["folder"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - chown(f, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed"), e) diff --git a/module/plugins/hooks/HotFolder.py b/module/plugins/hooks/HotFolder.py deleted file mode 100644 index ee1031ad5..000000000 --- a/module/plugins/hooks/HotFolder.py +++ /dev/null @@ -1,85 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN - @interface-version: 0.2 -""" - -from os import makedirs -from os import listdir -from os.path import exists -from os.path import join -from os.path import isfile -from shutil import move -import time - -from module.plugins.Hook import Hook - -class HotFolder(Hook): - __name__ = "HotFolder" - __version__ = "0.1" - __description__ = """observe folder and file for changes and add container and links""" - __config__ = [ ("activated", "bool", "Activated" , "False"), - ("folder", "str", "Folder to observe", "container"), - ("watch_file", "bool", "Observe link file", "False"), - ("keep", "bool", "Keep added containers", "True"), - ("file", "str", "Link file", "links.txt")] - __threaded__ = [] - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.de") - - def setup(self): - self.interval = 10 - - def periodical(self): - - if not exists(join(self.getConfig("folder"), "finished")): - makedirs(join(self.getConfig("folder"), "finished")) - - if self.getConfig("watch_file"): - - if not exists(self.getConfig("file")): - f = open(self.getConfig("file"), "wb") - f.close() - - - f = open(self.getConfig("file"), "rb") - content = f.read().strip() - f.close() - f = open(self.getConfig("file"), "wb") - f.close() - if content: - name = "%s_%s.txt" % (self.getConfig("file"), time.strftime("%H-%M-%S_%d%b%Y") ) - - f = open(join(self.getConfig("folder"), "finished", name), "wb") - f.write(content) - f.close() - - self.core.api.addPackage(f.name, [f.name], 1) - - for f in listdir(self.getConfig("folder")): - path = join(self.getConfig("folder"), f) - - if not isfile(path) or f.endswith("~") or f.startswith("#") or f.startswith("."): - continue - - newpath = join(self.getConfig("folder"), "finished", f if self.getConfig("keep") else "tmp_"+f) - move(path, newpath) - - self.log.info(_("Added %s from HotFolder") % f) - self.core.api.addPackage(f, [newpath], 1) - - \ No newline at end of file diff --git a/module/plugins/hooks/IRCInterface.py b/module/plugins/hooks/IRCInterface.py deleted file mode 100644 index e2737dc3a..000000000 --- a/module/plugins/hooks/IRCInterface.py +++ /dev/null @@ -1,425 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN - @author: jeix - @interface-version: 0.2 -""" - -from select import select -import socket -from threading import Thread -import time -from time import sleep -from traceback import print_exc -import re - -from module.plugins.Hook import Hook -from module.network.RequestFactory import getURL -from module.utils import formatSize - -from pycurl import FORM_FILE - -class IRCInterface(Thread, Hook): - __name__ = "IRCInterface" - __version__ = "0.1" - __description__ = """connect to irc and let owner perform different tasks""" - __config__ = [("activated", "bool", "Activated", "False"), - ("host", "str", "IRC-Server Address", "Enter your server here!"), - ("port", "int", "IRC-Server Port", "6667"), - ("ident", "str", "Clients ident", "pyload-irc"), - ("realname", "str", "Realname", "pyload-irc"), - ("nick", "str", "Nickname the Client will take", "pyLoad-IRC"), - ("owner", "str", "Nickname the Client will accept commands from", "Enter your nick here!"), - ("info_file", "bool", "Inform about every file finished", "False"), - ("info_pack", "bool", "Inform about every package finished", "True"), - ("captcha", "bool", "Send captcha requests", "True")] - __author_name__ = ("Jeix") - __author_mail__ = ("Jeix@hasnomail.com") - - def __init__(self, core, manager): - Thread.__init__(self) - Hook.__init__(self, core, manager) - self.setDaemon(True) - # self.sm = core.server_methods - self.api = core.api #todo, only use api - - def coreReady(self): - self.new_package = {} - - self.abort = False - - self.links_added = 0 - self.more = [] - - self.start() - - - def packageFinished(self, pypack): - try: - if self.getConfig("info_pack"): - self.response(_("Package finished: %s") % pypack.name) - except: - pass - - def downloadFinished(self, pyfile): - try: - if self.getConfig("info_file"): - self.response(_("Download finished: %(name)s @ %(plugin)s ") % { "name" : pyfile.name, "plugin": pyfile.pluginname} ) - except: - pass - - def newCaptchaTask(self, task): - if self.getConfig("captcha") and task.isTextual(): - task.handler.append(self) - task.setWaiting(60) - - page = getURL("http://www.freeimagehosting.net/upload.php", post={"attached" : (FORM_FILE, task.captchaFile)}, multipart=True) - - url = re.search(r"\[img\]([^\[]+)\[/img\]\[/url\]", page).group(1) - self.response(_("New Captcha Request: %s") % url) - self.response(_("Answer with 'c %s text on the captcha'") % task.id) - - def run(self): - # connect to IRC etc. - self.sock = socket.socket() - host = self.getConfig("host") - self.sock.connect((host, self.getConfig("port"))) - nick = self.getConfig("nick") - self.sock.send("NICK %s\r\n" % nick) - self.sock.send("USER %s %s bla :%s\r\n" % (nick, host, nick)) - for t in self.getConfig("owner").split(): - if t.strip().startswith("#"): - self.sock.send("JOIN %s\r\n" % t.strip()) - self.log.info("pyLoad IRC: Connected to %s!" % host) - self.log.info("pyLoad IRC: Switching to listening mode!") - try: - self.main_loop() - - except IRCError, ex: - self.sock.send("QUIT :byebye\r\n") - print_exc() - self.sock.close() - - - def main_loop(self): - readbuffer = "" - while True: - sleep(1) - fdset = select([self.sock], [], [], 0) - if self.sock not in fdset[0]: - continue - - if self.abort: - raise IRCError("quit") - - readbuffer += self.sock.recv(1024) - temp = readbuffer.split("\n") - readbuffer = temp.pop() - - for line in temp: - line = line.rstrip() - first = line.split() - - if first[0] == "PING": - self.sock.send("PONG %s\r\n" % first[1]) - - if first[0] == "ERROR": - raise IRCError(line) - - msg = line.split(None, 3) - if len(msg) < 4: - continue - - msg = { - "origin":msg[0][1:], - "action":msg[1], - "target":msg[2], - "text":msg[3][1:] - } - - self.handle_events(msg) - - - def handle_events(self, msg): - if not msg["origin"].split("!", 1)[0] in self.getConfig("owner").split(): - return - - if msg["target"].split("!", 1)[0] != self.getConfig("nick"): - return - - if msg["action"] != "PRIVMSG": - return - - # HANDLE CTCP ANTI FLOOD/BOT PROTECTION - if msg["text"] == "\x01VERSION\x01": - self.log.debug("Sending CTCP VERSION.") - self.sock.send("NOTICE %s :%s\r\n" % (msg['origin'], "pyLoad! IRC Interface")) - return - elif msg["text"] == "\x01TIME\x01": - self.log.debug("Sending CTCP TIME.") - self.sock.send("NOTICE %s :%d\r\n" % (msg['origin'], time.time())) - return - elif msg["text"] == "\x01LAG\x01": - self.log.debug("Received CTCP LAG.") # don't know how to answer - return - - trigger = "pass" - args = None - - try: - temp = msg["text"].split() - trigger = temp[0] - if len(temp) > 1: - args = temp[1:] - except: - pass - - handler = getattr(self, "event_%s" % trigger, self.event_pass) - try: - res = handler(args) - for line in res: - self.response(line, msg["origin"]) - except Exception, e: - self.log.error("pyLoad IRC: "+ repr(e)) - - - def response(self, msg, origin=""): - if origin == "": - for t in self.getConfig("owner").split(): - self.sock.send("PRIVMSG %s :%s\r\n" % (t.strip(), msg)) - else: - self.sock.send("PRIVMSG %s :%s\r\n" % (origin.split("!", 1)[0], msg)) - - -#### Events - def event_pass(self, args): - return [] - - def event_status(self, args): - downloads = self.api.statusDownloads() - if not downloads: - return ["INFO: There are no active downloads currently."] - - temp_progress = "" - lines = ["ID - Name - Status - Speed - ETA - Progress"] - for data in downloads: - - if data.status == 5: - temp_progress = data.format_wait - else: - temp_progress = "%d%% (%s)" % (data.percent, data.format_size) - - lines.append("#%d - %s - %s - %s - %s - %s" % - ( - data.fid, - data.name, - data.statusmsg, - "%s/s" % formatSize(data.speed), - "%s" % data.format_eta, - temp_progress - ) - ) - return lines - - def event_queue(self, args): - ps = self.api.getQueue() - - if not ps: - return ["INFO: There are no packages in queue."] - - lines = [] - for pack in ps: - lines.append('PACKAGE #%s: "%s" with %d links.' % (pack.pid, pack.name, len(pack.fids) )) - - return lines - - def event_collector(self, args): - ps = self.api.getCollector() - if not ps: - return ["INFO: No packages in collector!"] - - lines = [] - for pack in ps: - lines.append('PACKAGE #%s: "%s" with %d links.' % (pack.pid, pack.name, len(pack.fids) )) - - return lines - - def event_info(self, args): - if not args: - return ['ERROR: Use info like this: info '] - - info = self.api.getFileData(int(args[0])) - - if not info: - return ["ERROR: Link doesn't exists."] - - return ['LINK #%s: %s (%s) [%s][%s]' % (info.fid, info.name, info.format_size, info.status_msg, - info.plugin)] - - def event_packinfo(self, args): - if not args: - return ['ERROR: Use packinfo like this: packinfo '] - - lines = [] - pack = self.api.getPackageData(int(args[0])) - - if not pack: - return ["ERROR: Package doesn't exists."] - - id = args[0] - - self.more = [] - - lines.append('PACKAGE #%s: "%s" with %d links' % (id, pack.name, len(pack.links)) ) - for pyfile in pack.links: - self.more.append('LINK #%s: %s (%s) [%s][%s]' % (pyfile.fid, pyfile.name, pyfile.format_size, - pyfile.statusmsg, pyfile.plugin)) - - if len(self.more) < 6: - lines.extend(self.more) - self.more = [] - else: - lines.extend(self.more[:6]) - self.more = self.more[6:] - lines.append("%d more links do display." % len(self.more)) - - - return lines - - def event_more(self, args): - if not self.more: - return ["No more information to display."] - - lines = self.more[:6] - self.more = self.more[6:] - lines.append("%d more links do display." % len(self.more)) - - return lines - - def event_start(self, args): - - self.api.unpauseServer() - return ["INFO: Starting downloads."] - - def event_stop(self, args): - - self.api.pauseServer() - return ["INFO: No new downloads will be started."] - - - def event_add(self, args): - if len(args) < 2: - return ['ERROR: Add links like this: "add links". ', - 'This will add the link to to the package / the package with id !'] - - - - pack = args[0].strip() - links = [x.strip() for x in args[1:]] - - count_added = 0 - count_failed = 0 - try: - id = int(pack) - pack = self.api.getPackageData(id) - if not pack: - return ["ERROR: Package doesn't exists."] - - #TODO add links - - return ["INFO: Added %d links to Package %s [#%d]" % (len(links), pack["name"], id)] - - except: - # create new package - id = self.api.addPackage(pack, links, 1) - return ["INFO: Created new Package %s [#%d] with %d links." % (pack, id, len(links))] - - - def event_del(self, args): - if len(args) < 2: - return ["ERROR: Use del command like this: del -p|-l [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)"] - - if args[0] == "-p": - ret = self.api.deletePackages(map(int, args[1:])) - return ["INFO: Deleted %d packages!" % len(args[1:])] - - elif args[0] == "-l": - ret = self.api.delLinks(map(int, args[1:])) - return ["INFO: Deleted %d links!" % len(args[1:])] - - else: - return ["ERROR: Use del command like this: del <-p|-l> [...] (-p indicates that the ids are from packages, -l indicates that the ids are from links)"] - - def event_push(self, args): - if not args: - return ["ERROR: Push package to queue like this: push "] - - id = int(args[0]) - if not self.api.getPackage_data(id): - return ["ERROR: Package #%d does not exist." % id] - - self.api.pushToQueue(id) - return ["INFO: Pushed package #%d to queue." % id] - - def event_pull(self, args): - if not args: - return ["ERROR: Pull package from queue like this: pull ."] - - id = int(args[0]) - if not self.api.getPackageData(id): - return ["ERROR: Package #%d does not exist." % id] - - self.api.pullFromQueue(id) - return ["INFO: Pulled package #%d from queue to collector." % id] - - def event_c(self, args): - """ captcha answer """ - if not args: - return ["ERROR: Captcha ID missing."] - - task = self.core.captchaManager.getTaskByID(args[0]) - if not task: - return ["ERROR: Captcha Task with ID %s does not exists." % args[0]] - - task.setResult(" ".join(args[1:])) - return ["INFO: Result %s saved." % " ".join(args[1:])] - - - def event_help(self, args): - lines = ["The following commands are available:", - "add [...] Adds link to package. (creates new package if it does not exist)", - "queue Shows all packages in the queue", - "collector Shows all packages in collector", - "del -p|-l [...] Deletes all packages|links with the ids specified", - "info Shows info of the link with id ", - "packinfo Shows info of the package with id ", - "more Shows more info when the result was truncated", - "start Starts all downloads", - "stop Stops the download (but not abort active downloads)", - "push Push package to queue", - "pull Pull package from queue", - "status Show general download status", - "help Shows this help message"] - return lines - - -class IRCError(Exception): - def __init__(self, value): - Exception.__init__(value) - self.value = value - def __str__(self): - return repr(self.value) diff --git a/module/plugins/hooks/MergeFiles.py b/module/plugins/hooks/MergeFiles.py deleted file mode 100644 index 02d343096..000000000 --- a/module/plugins/hooks/MergeFiles.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: and9000 -""" - -import os -import re -import sys -import traceback - -from os.path import join -from module.utils import save_join, fs_encode -from module.plugins.Hook import Hook - -BUFFER_SIZE = 4096 - -class MergeFiles(Hook): - __name__ = "MergeFiles" - __version__ = "0.1" - __description__ = "Merges parts splitted with hjsplit" - __config__ = [ - ("activated" , "bool" , "Activated" , "False"), - ] - __threaded__ = ["packageFinished"] - __author_name__ = ("and9000") - __author_mail__ = ("me@has-no-mail.com") - - def setup(self): - # nothing to do - pass - - def packageFinished(self, pack): - files = {} - fid_dict = {} - for fid, data in pack.getChildren().iteritems(): - if re.search("\.[0-9]{3}$", data["name"]): - if data["name"][:-4] not in files: - files[data["name"][:-4]] = [] - files[data["name"][:-4]].append(data["name"]) - files[data["name"][:-4]].sort() - fid_dict[data["name"]] = fid - - download_folder = self.core.config['general']['download_folder'] - - if self.core.config['general']['folder_per_package']: - download_folder = save_join(download_folder, pack.folder) - - for name, file_list in files.iteritems(): - self.core.log.info("Starting merging of %s" % name) - final_file = open(join(download_folder, fs_encode(name)), "wb") - - for splitted_file in file_list: - self.core.log.debug("Merging part %s" % splitted_file) - pyfile = self.core.files.getFile(fid_dict[splitted_file]) - pyfile.setStatus("processing") - try: - s_file = open(os.path.join(download_folder, splitted_file), "rb") - size_written = 0 - s_file_size = int(os.path.getsize(os.path.join(download_folder, splitted_file))) - while True: - f_buffer = s_file.read(BUFFER_SIZE) - if f_buffer: - final_file.write(f_buffer) - size_written += BUFFER_SIZE - pyfile.setProgress((size_written*100)/s_file_size) - else: - break - s_file.close() - self.core.log.debug("Finished merging part %s" % splitted_file) - except Exception, e: - print traceback.print_exc() - finally: - pyfile.setProgress(100) - pyfile.setStatus("finished") - pyfile.release() - - final_file.close() - self.core.log.info("Finished merging of %s" % name) - - diff --git a/module/plugins/hooks/MultiHome.py b/module/plugins/hooks/MultiHome.py deleted file mode 100644 index f15148538..000000000 --- a/module/plugins/hooks/MultiHome.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from module.plugins.Hook import Hook -from time import time - -class MultiHome(Hook): - __name__ = "MultiHome" - __version__ = "0.1" - __description__ = """ip address changer""" - __config__ = [ ("activated", "bool", "Activated" , "False"), - ("interfaces", "str", "Interfaces" , "None") ] - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def setup(self): - self.register = {} - self.interfaces = [] - self.parseInterfaces(self.getConfig("interfaces").split(";")) - if not self.interfaces: - self.parseInterfaces([self.config["download"]["interface"]]) - self.setConfig("interfaces", self.toConfig()) - - def toConfig(self): - return ";".join([i.adress for i in self.interfaces]) - - def parseInterfaces(self, interfaces): - for interface in interfaces: - if not interface or str(interface).lower() == "none": - continue - self.interfaces.append(Interface(interface)) - - def coreReady(self): - requestFactory = self.core.requestFactory - oldGetRequest = requestFactory.getRequest - def getRequest(pluginName, account=None): - iface = self.bestInterface(pluginName, account) - if iface: - iface.useFor(pluginName, account) - requestFactory.iface = lambda: iface.adress - self.log.debug("Multihome: using address: "+iface.adress) - return oldGetRequest(pluginName, account) - requestFactory.getRequest = getRequest - - def bestInterface(self, pluginName, account): - best = None - for interface in self.interfaces: - if not best or interface.lastPluginAccess(pluginName, account) < best.lastPluginAccess(pluginName, account): - best = interface - return best - -class Interface(object): - def __init__(self, adress): - self.adress = adress - self.history = {} - - def lastPluginAccess(self, pluginName, account): - if (pluginName, account) in self.history: - return self.history[(pluginName, account)] - return 0 - - def useFor(self, pluginName, account): - self.history[(pluginName, account)] = time() - - def __repr__(self): - return "" % self.adress diff --git a/module/plugins/hooks/MultiHoster.py b/module/plugins/hooks/MultiHoster.py deleted file mode 100644 index 2a567cce4..000000000 --- a/module/plugins/hooks/MultiHoster.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re -from types import MethodType - -from module.plugins.MultiHoster import MultiHoster as MultiHosterAccount, normalize -from module.plugins.Hook import Hook, AddEventListener -from module.plugins.PluginManager import PluginTuple - -class MultiHoster(Hook): - __version__ = "0.1" - __internal__ = True - __description__ = "Gives ability to use MultiHoster services. You need to add your account first." - __config__ = [] - __author_mail__ = ("pyLoad Team",) - __author_mail__ = ("support@pyload.org",) - - #TODO: multiple accounts - multihoster / config options - - def init(self): - - # overwritten plugins - self.plugins = {} - - def addHoster(self, account): - - self.logDebug("New MultiHoster %s" % account.__name__) - - pluginMap = {} - for name in self.core.pluginManager.getPlugins("hoster").keys(): - pluginMap[name.lower()] = name - - supported = [] - new_supported = [] - - for hoster in account.getHosterList(): - name = normalize(hoster) - - if name in pluginMap: - supported.append(pluginMap[name]) - else: - new_supported.append(hoster) - - if not supported and not new_supported: - account.logError(_("No Hoster loaded")) - return - - klass = self.core.pluginManager.getPluginClass(account.__name__) - - # inject plugin plugin - account.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(supported))) - for hoster in supported: - self.plugins[hoster] = klass - - account.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) - - # create new regexp - regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) - - # recreate plugin tuple for new regexp - hoster = self.core.pluginManager.getPlugins("hoster") - p = hoster[account.__name__] - new = PluginTuple(p.version, re.compile(regexp), p.deps, p.user, p.path) - hoster[account.__name__] = new - - - - @AddEventListener("accountDeleted") - def refreshAccounts(self, plugin=None, user=None): - - self.plugins = {} - - for name, account in self.core.accountManager.iterAccounts(): - if isinstance(account, MultiHosterAccount) and account.isUsable(): - self.addHoster(account) - - @AddEventListener("accountUpdated") - def refreshAccount(self, plugin, user): - - account = self.core.accountManager.getAccount(plugin, user) - if isinstance(account, MultiHosterAccount) and account.isUsable(): - self.addHoster(account) - - def activate(self): - self.refreshAccounts() - - # new method for plugin manager - def getPlugin(self2, name): - if name in self.plugins: - return self.plugins[name] - return self2.getPluginClass(name) - - pm = self.core.pluginManager - pm.getPlugin = MethodType(getPlugin, pm, object) - - - def deactivate(self): - #restore state - pm = self.core.pluginManager - pm.getPlugin = pm.getPluginClass - diff --git a/module/plugins/hooks/MultishareCz.py b/module/plugins/hooks/MultishareCz.py deleted file mode 100644 index a934f43ef..000000000 --- a/module/plugins/hooks/MultishareCz.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.network.RequestFactory import getURL -from module.plugins.internal.MultiHoster import MultiHoster -import re - -def getConfigSet(option): - s = set(option.lower().split('|')) - s.discard(u'') - return s - -class MultishareCz(MultiHoster): - __name__ = "MultishareCz" - __version__ = "0.01" - __type__ = "hook" - __config__ = [("activated", "bool", "Activated", "False"), - ("includeHoster", "str", "Use only for downloads from (bar-separated hosters)", ""), - ("excludeHoster", "str", "Do not use for downloads from (bar-separated hosters)", "rapidshare.com|uloz.to")] - __description__ = """MultiShare.cz hook plugin""" - __author_name__ = ("zoidberg") - __author_mail__ = ("zoidberg@mujmail.cz") - - #replacements = [("freakshare.net", "freakshare.com")] - HOSTER_PATTERN = r']*alt="([^"]+)">\s*OK' - - def getHoster(self): - - page = getURL("http://www.multishare.cz/monitoring/") - hoster = set(m.group(1).lower() for m in re.finditer(self.HOSTER_PATTERN, page)) - - option = self.getConfig('includeHoster').strip() - if option: hoster &= getConfigSet(option) - option = self.getConfig('excludeHoster').strip() - if option: hoster -= getConfigSet(option) - - return list(hoster) \ No newline at end of file diff --git a/module/plugins/hooks/Premium4Me.py b/module/plugins/hooks/Premium4Me.py deleted file mode 100644 index fc3ce2343..000000000 --- a/module/plugins/hooks/Premium4Me.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.network.RequestFactory import getURL -from module.plugins.internal.MultiHoster import MultiHoster - -class Premium4Me(MultiHoster): - __name__ = "Premium4Me" - __version__ = "0.02" - __type__ = "hook" - - __config__ = [("activated", "bool", "Activated", "False"), - ("hosterListMode", "all;listed;unlisted", "Use for downloads from supported hosters:", "all"), - ("hosterList", "str", "Hoster list (comma separated)", "")] - __description__ = """premium4.me hook plugin""" - __author_name__ = ("RaNaN", "zoidberg") - __author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz") - - replacements = [("freakshare.net", "freakshare.com")] - - def getHoster(self): - - page = getURL("http://premium4.me/api/hosters.php?authcode=%s" % self.account.authcode) - hosters = set([x.strip() for x in page.replace("\"", "").split(";")]) - - configMode = self.getConfig('hosterListMode') - if configMode in ("listed", "unlisted"): - configList = set(self.getConfig('hosterList').strip().lower().replace(',','|').split('|')) - configList.discard(u'') - if configMode == "listed": - hosters &= configList - elif configMode == "unlisted": - hosters -= configList - - return list(hosters) - - def coreReady(self): - - self.account = self.core.accountManager.getAccountPlugin("Premium4Me") - - user = self.account.selectAccount()[0] - - if not user: - self.logError(_("Please add your premium4.me account first and restart pyLoad")) - return - - return MultiHoster.coreReady(self) \ No newline at end of file diff --git a/module/plugins/hooks/RehostTo.py b/module/plugins/hooks/RehostTo.py deleted file mode 100644 index b16987f5c..000000000 --- a/module/plugins/hooks/RehostTo.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.network.RequestFactory import getURL -from module.plugins.internal.MultiHoster import MultiHoster - -class RehostTo(MultiHoster): - __name__ = "RehostTo" - __version__ = "0.41" - __type__ = "hook" - - __config__ = [("activated", "bool", "Activated", "False")] - - __description__ = """rehost.to hook plugin""" - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - replacements = [("freakshare.net", "freakshare.com")] - - def getHoster(self): - - page = getURL("http://rehost.to/api.php?cmd=get_supported_och_dl&long_ses=%s" % self.long_ses) - return [x.strip() for x in page.replace("\"", "").split(",")] - - - def coreReady(self): - - self.account = self.core.accountManager.getAccountPlugin("RehostTo") - - user = self.account.selectAccount()[0] - - if not user: - self.log.error("Rehost.to: "+ _("Please add your rehost.to account first and restart pyLoad")) - return - - data = self.account.getAccountInfo(user) - self.ses = data["ses"] - self.long_ses = data["long_ses"] - - return MultiHoster.coreReady(self) diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py deleted file mode 100644 index 230a6e858..000000000 --- a/module/plugins/hooks/UpdateManager.py +++ /dev/null @@ -1,199 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -import sys -import re -from os import stat -from os.path import join, exists -from time import time - -from module.plugins.PluginManager import IGNORE -from module.network.RequestFactory import getURL -from module.plugins.Hook import threaded, Expose, Hook - -class UpdateManager(Hook): - __name__ = "UpdateManager" - __version__ = "0.12" - __description__ = """checks for updates""" - __config__ = [("activated", "bool", "Activated", "True"), - ("interval", "int", "Check interval in minutes", "360"), - ("debug", "bool", "Check for plugin changes when in debug mode", False)] - __author_name__ = ("RaNaN") - __author_mail__ = ("ranan@pyload.org") - - @property - def debug(self): - return self.core.debug and self.getConfig("debug") - - - def setup(self): - if self.debug: - self.logDebug("Monitoring file changes") - self.interval = 4 - self.last_check = 0 #timestamp of updatecheck - self.old_periodical = self.periodical - self.periodical = self.checkChanges - self.mtimes = {} #recordes times - else: - self.interval = self.getConfig("interval") * 60 - - self.updated = False - self.reloaded = True - - self.info = {"pyload": False, "plugins": False} - - @threaded - def periodical(self): - - if self.core.version.endswith("-dev"): - self.logDebug("No update check performed on dev version.") - return - - update = self.checkForUpdate() - if update: - self.info["pyload"] = True - else: - self.log.info(_("No Updates for pyLoad")) - self.checkPlugins() - - if self.updated and not self.reloaded: - self.info["plugins"] = True - self.log.info(_("*** Plugins have been updated, please restart pyLoad ***")) - elif self.updated and self.reloaded: - self.log.info(_("Plugins updated and reloaded")) - self.updated = False - else: - self.log.info(_("No plugin updates available")) - - @Expose - def recheckForUpdates(self): - """recheck if updates are available""" - self.periodical() - - def checkForUpdate(self): - """checks if an update is available""" - - try: - version_check = getURL("http://get.pyload.org/check/%s/" % self.core.api.getServerVersion()) - if version_check == "": - return False - else: - self.log.info(_("*** New pyLoad Version %s available ***") % version_check) - self.log.info(_("*** Get it here: http://pyload.org/download ***")) - return True - except: - self.log.warning(_("Not able to connect server for updates")) - return False - - - def checkPlugins(self): - """ checks for plugins updates""" - - # plugins were already updated - if self.info["plugins"]: return - - try: - updates = getURL("http://get.pyload.org/plugins/check/") - except: - self.log.warning(_("Not able to connect server for updates")) - return False - - updates = updates.splitlines() - reloads = [] - - vre = re.compile(r'__version__.*=.*("|\')([0-9.]+)') - - for plugin in updates: - path, version = plugin.split(":") - prefix, filename = path.split("/") - - if filename.endswith(".pyc"): - name = filename[:filename.find("_")] - else: - name = filename.replace(".py", "") - - #TODO: obsolete - if prefix.endswith("s"): - type = prefix[:-1] - else: - type = prefix - - plugins = self.core.pluginManager.getPlugins(type) - - if name in plugins: - if float(plugins[name].version) >= float(version): - continue - - if name in IGNORE or (type, name) in IGNORE: - continue - - self.log.info(_("New version of %(type)s|%(name)s : %(version).2f") % { - "type": type, - "name": name, - "version": float(version) - }) - - try: - content = getURL("http://get.pyload.org/plugins/get/" + path) - except Exception, e: - self.logWarning(_("Error when updating %s") % filename, str(e)) - continue - - m = vre.search(content) - if not m or m.group(2) != version: - self.logWarning(_("Error when updating %s") % name, _("Version mismatch")) - continue - - f = open(join("userplugins", prefix, filename), "wb") - f.write(content) - f.close() - self.updated = True - - reloads.append((prefix, name)) - - self.reloaded = self.core.pluginManager.reloadPlugins(reloads) - - def checkChanges(self): - - if self.last_check + self.getConfig("interval") * 60 < time(): - self.old_periodical() - self.last_check = time() - - modules = filter( - lambda m: m and (m.__name__.startswith("module.plugins.") or m.__name__.startswith("userplugins.")) and m.__name__.count(".") >= 2, - sys.modules.itervalues()) - - reloads = [] - - for m in modules: - root, type, name = m.__name__.rsplit(".", 2) - id = (type, name) - if type in self.core.pluginManager.plugins: - f = m.__file__.replace(".pyc", ".py") - if not exists(f): continue - - mtime = stat(f).st_mtime - - if id not in self.mtimes: - self.mtimes[id] = mtime - elif self.mtimes[id] < mtime: - reloads.append(id) - self.mtimes[id] = mtime - - self.core.pluginManager.reloadPlugins(reloads) diff --git a/module/plugins/hooks/XMPPInterface.py b/module/plugins/hooks/XMPPInterface.py deleted file mode 100644 index de87433cf..000000000 --- a/module/plugins/hooks/XMPPInterface.py +++ /dev/null @@ -1,276 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN - @interface-version: 0.2 -""" - -from pyxmpp import streamtls -from pyxmpp.all import JID, Message, Presence -from pyxmpp.jabber.client import JabberClient -from pyxmpp.interface import implements -from pyxmpp.interfaces import * - -from module.plugins.hooks.IRCInterface import IRCInterface - -class XMPPInterface(IRCInterface, JabberClient): - __name__ = "XMPPInterface" - __version__ = "0.1" - __description__ = """connect to jabber and let owner perform different tasks""" - __config__ = [("activated", "bool", "Activated", "False"), - ("jid", "str", "Jabber ID", "user@exmaple-jabber-server.org"), - ("pw", "str", "Password", ""), - ("tls", "bool", "Use TLS", False), - ("owners", "str", "List of JIDs accepting commands from", "me@icq-gateway.org;some@msn-gateway.org"), - ("info_file", "bool", "Inform about every file finished", "False"), - ("info_pack", "bool", "Inform about every package finished", "True"), - ("captcha", "bool", "Send captcha requests", "True")] - __author_name__ = ("RaNaN") - __author_mail__ = ("RaNaN@pyload.org") - - implements(IMessageHandlersProvider) - - def __init__(self, core, manager): - IRCInterface.__init__(self, core, manager) - - self.jid = JID(self.getConfig("jid")) - password = self.getConfig("pw") - - # if bare JID is provided add a resource -- it is required - if not self.jid.resource: - self.jid = JID(self.jid.node, self.jid.domain, "pyLoad") - - if self.getConfig("tls"): - tls_settings = streamtls.TLSSettings(require=True, verify_peer=False) - auth = ("sasl:PLAIN", "sasl:DIGEST-MD5") - else: - tls_settings = None - auth = ("sasl:DIGEST-MD5", "digest") - - # setup client with provided connection information - # and identity data - JabberClient.__init__(self, self.jid, password, - disco_name="pyLoad XMPP Client", disco_type="bot", - tls_settings=tls_settings, auth_methods=auth) - - self.interface_providers = [ - VersionHandler(self), - self, - ] - - def coreReady(self): - self.new_package = {} - - self.start() - - def packageFinished(self, pypack): - try: - if self.getConfig("info_pack"): - self.announce(_("Package finished: %s") % pypack.name) - except: - pass - - def downloadFinished(self, pyfile): - try: - if self.getConfig("info_file"): - self.announce( - _("Download finished: %(name)s @ %(plugin)s") % {"name": pyfile.name, "plugin": pyfile.pluginname}) - except: - pass - - def run(self): - # connect to IRC etc. - self.connect() - try: - self.loop() - except Exception, ex: - self.core.log.error("pyLoad XMPP: %s" % str(ex)) - - def stream_state_changed(self, state, arg): - """This one is called when the state of stream connecting the component - to a server changes. This will usually be used to let the user - know what is going on.""" - self.log.debug("pyLoad XMPP: *** State changed: %s %r ***" % (state, arg)) - - def disconnected(self): - self.log.debug("pyLoad XMPP: Client was disconnected") - - def stream_closed(self, stream): - self.log.debug("pyLoad XMPP: Stream was closed | %s" % stream) - - def stream_error(self, err): - self.log.debug("pyLoad XMPP: Stream Error: %s" % err) - - def get_message_handlers(self): - """Return list of (message_type, message_handler) tuples. - - The handlers returned will be called when matching message is received - in a client session.""" - return [ - ("normal", self.message), - ] - - def presence_control(self, stanza): - from_jid = unicode(stanza.get_from_jid()) - stanza_type = stanza.get_type() - self.log.debug("pyLoad XMPP: %s stanza from %s" % (stanza_type, - from_jid)) - - if from_jid in self.getConfig("owners"): - return stanza.make_accept_response() - - return stanza.make_deny_response() - - def session_started(self): - self.stream.send(Presence()) - - self.stream.set_presence_handler("subscribe", self.presence_control) - self.stream.set_presence_handler("subscribed", self.presence_control) - self.stream.set_presence_handler("unsubscribe", self.presence_control) - self.stream.set_presence_handler("unsubscribed", self.presence_control) - - def message(self, stanza): - """Message handler for the component.""" - subject = stanza.get_subject() - body = stanza.get_body() - t = stanza.get_type() - self.log.debug(u'pyLoad XMPP: Message from %s received.' % (unicode(stanza.get_from(), ))) - self.log.debug(u'pyLoad XMPP: Body: %s Subject: %s Type: %s' % (body, subject, t)) - - if t == "headline": - # 'headline' messages should never be replied to - return True - if subject: - subject = u"Re: " + subject - - to_jid = stanza.get_from() - from_jid = stanza.get_to() - - #j = JID() - to_name = to_jid.as_utf8() - from_name = from_jid.as_utf8() - - names = self.getConfig("owners").split(";") - - if to_name in names or to_jid.node + "@" + to_jid.domain in names: - messages = [] - - trigger = "pass" - args = None - - try: - temp = body.split() - trigger = temp[0] - if len(temp) > 1: - args = temp[1:] - except: - pass - - handler = getattr(self, "event_%s" % trigger, self.event_pass) - try: - res = handler(args) - for line in res: - m = Message( - to_jid=to_jid, - from_jid=from_jid, - stanza_type=stanza.get_type(), - subject=subject, - body=line) - - messages.append(m) - except Exception, e: - self.log.error("pyLoad XMPP: " + repr(e)) - - return messages - - else: - return True - - def response(self, msg, origin=""): - return self.announce(msg) - - def announce(self, message): - """ send message to all owners""" - for user in self.getConfig("owners").split(";"): - self.log.debug("pyLoad XMPP: Send message to %s" % user) - - to_jid = JID(user) - - m = Message(from_jid=self.jid, - to_jid=to_jid, - stanza_type="chat", - body=message) - - stream = self.get_stream() - if not stream: - self.connect() - stream = self.get_stream() - - stream.send(m) - - def beforeReconnecting(self, ip): - self.disconnect() - - def afterReconnecting(self, ip): - self.connect() - - -class VersionHandler(object): - """Provides handler for a version query. - - This class will answer version query and announce 'jabber:iq:version' namespace - in the client's disco#info results.""" - - implements(IIqHandlersProvider, IFeaturesProvider) - - def __init__(self, client): - """Just remember who created this.""" - self.client = client - - def get_features(self): - """Return namespace which should the client include in its reply to a - disco#info query.""" - return ["jabber:iq:version"] - - def get_iq_get_handlers(self): - """Return list of tuples (element_name, namespace, handler) describing - handlers of stanzas""" - return [ - ("query", "jabber:iq:version", self.get_version), - ] - - def get_iq_set_handlers(self): - """Return empty list, as this class provides no stanza handler.""" - return [] - - def get_version(self, iq): - """Handler for jabber:iq:version queries. - - jabber:iq:version queries are not supported directly by PyXMPP, so the - XML node is accessed directly through the libxml2 API. This should be - used very carefully!""" - iq = iq.make_result_response() - q = iq.new_query("jabber:iq:version") - q.newTextChild(q.ns(), "name", "Echo component") - q.newTextChild(q.ns(), "version", "1.0") - return iq - - def unload(self): - self.log.debug("pyLoad XMPP: unloading") - self.disconnect() - - def deactivate(self): - self.unload() diff --git a/module/plugins/hooks/__init__.py b/module/plugins/hooks/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index 2130f910e..ceb188193 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -30,7 +30,7 @@ class AbtractExtractor: def __init__(self, m, file, out, fullpath, overwrite, renice): """Initialize extractor for specific file - :param m: ExtractArchive Hook plugin + :param m: ExtractArchive Addon plugin :param file: Absolute filepath :param out: Absolute path to destination directory :param fullpath: extract to fullpath -- cgit v1.2.3 From 50d4df8b4d48b855bd18e9922355b7f3f2b4da4e Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 20 Mar 2012 14:57:45 +0100 Subject: captcha decrypting for all plugin types, new interaction manager --- module/plugins/Base.py | 118 +++++++++- module/plugins/Hoster.py | 96 +------- module/plugins/captcha/GigasizeCom.py | 19 -- module/plugins/captcha/LinksaveIn.py | 147 ------------- module/plugins/captcha/MegauploadCom.py | 14 -- module/plugins/captcha/NetloadIn.py | 24 -- module/plugins/captcha/ShareonlineBiz.py | 53 ----- module/plugins/captcha/__init__.py | 0 module/plugins/captcha/captcha.py | 315 --------------------------- module/plugins/internal/NetloadInOCR.py | 27 +++ module/plugins/internal/OCR.py | 314 ++++++++++++++++++++++++++ module/plugins/internal/ShareonlineBizOCR.py | 53 +++++ 12 files changed, 507 insertions(+), 673 deletions(-) delete mode 100644 module/plugins/captcha/GigasizeCom.py delete mode 100644 module/plugins/captcha/LinksaveIn.py delete mode 100644 module/plugins/captcha/MegauploadCom.py delete mode 100644 module/plugins/captcha/NetloadIn.py delete mode 100644 module/plugins/captcha/ShareonlineBiz.py delete mode 100644 module/plugins/captcha/__init__.py delete mode 100644 module/plugins/captcha/captcha.py create mode 100644 module/plugins/internal/NetloadInOCR.py create mode 100644 module/plugins/internal/OCR.py create mode 100644 module/plugins/internal/ShareonlineBizOCR.py (limited to 'module/plugins') diff --git a/module/plugins/Base.py b/module/plugins/Base.py index b846bbd60..61fa211f4 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -18,8 +18,11 @@ """ import sys +from time import time, sleep +from random import randint + from module.utils import decode -from module.utils.fs import exists, makedirs, join +from module.utils.fs import exists, makedirs, join, remove # TODO # more attributes if needed @@ -32,6 +35,9 @@ class Fail(Exception): class Retry(Exception): """ raised when start again from beginning """ +class Abort(Exception): + """ raised when aborted """ + class Base(object): """ The Base plugin class with all shared methods and every possible attribute for plugin definition. @@ -83,6 +89,9 @@ class Base(object): #: :class:`InteractionManager` self.im = core.interactionManager + #: last interaction task + self.task = None + def logInfo(self, *args, **kwargs): """ Print args to log at specific level @@ -118,11 +127,7 @@ class Base(object): getattr(self.log, level)("%s: %s" % (self.__name__, sep.join(strings))) def setConfig(self, option, value): - """ Set config value for current plugin - - :param option: - :param value: - """ + """ Set config value for current plugin """ self.core.config.set(self.__name__, option, value) def getConf(self, option): @@ -130,11 +135,7 @@ class Base(object): return self.core.config.get(self.__name__, option) def getConfig(self, option): - """ Returns config value for current plugin - - :param option: - :return: - """ + """ Returns config value for current plugin """ return self.getConf(option) def setStorage(self, key, value): @@ -167,6 +168,14 @@ class Base(object): sys.stdout = sys._stdout embed() + def abort(self): + """ Check if plugin is in an abort state, is overwritten by subtypes""" + return False + + def checkAbort(self): + """ Will be overwriten to determine if control flow should be aborted """ + if self.abort: raise Abort() + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """Load content at url and returns it @@ -180,6 +189,7 @@ class Base(object): :return: Loaded content """ if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") + self.checkAbort() res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) @@ -225,6 +235,92 @@ class Base(object): return res + def invalidTask(self): + if self.task: + self.task.invalid() + + def invalidCaptcha(self): + self.logDebug("Deprecated method .invalidCaptcha, use .invalidTask") + self.invalidTask() + + def correctTask(self): + if self.task: + self.task.correct() + + def correctCaptcha(self): + self.logDebug("Deprecated method .correctCaptcha, use .correctTask") + self.correctTask() + + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', + result_type='textual'): + """ Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param forceUser: if True, ocr is not used + :param imgtype: Type of the Image + :param result_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + + :return: result of decrypting + """ + + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time())[-6:].replace(".", "") + temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + temp_file.write(img) + temp_file.close() + + name = "%sOCR" % self.__name__ + has_plugin = name in self.core.pluginManager.getPlugins("internal") + + if self.core.captcha: + OCR = self.core.pluginManager.loadClass("internal", name) + else: + OCR = None + + if OCR and not forceUser: + sleep(randint(3000, 5000) / 1000.0) + self.checkAbort() + + ocr = OCR() + result = ocr.get_captcha(temp_file.name) + else: + task = self.im.newCaptchaTask(img, imgtype, temp_file.name, result_type) + self.task = task + self.im.handleTask(task) + + while task.isWaiting(): + if self.abort(): + self.im.removeTask(task) + raise Abort() + sleep(1) + + #TODO + self.im.removeTask(task) + + if task.error and has_plugin: #ignore default error message since the user could use OCR + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) + + result = task.result + self.log.debug("Received captcha result: %s" % str(result)) + + if not self.core.debug: + try: + remove(temp_file.name) + except: + pass + + return result + def fail(self, reason): """ fail and give reason """ raise Fail(reason) \ No newline at end of file diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 32c587aa5..b330743e6 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -17,10 +17,8 @@ @author: RaNaN, spoob, mkaay """ -from time import time, sleep -from random import randint - import os +from time import time if os.name != "nt": from module.utils.fs import chown @@ -35,9 +33,6 @@ from module.utils.fs import save_join, save_filename, fs_encode, fs_decode,\ # Import for Hoster Plugins chunks = _chunks -class Abort(Exception): - """ raised when aborted """ - class Reconnect(Exception): """ raised when reconnected """ @@ -170,6 +165,9 @@ class Hoster(Base): """the 'main' method of every plugin, you **have to** overwrite it""" raise NotImplementedError + def abort(self): + return self.pyfile.abort + def resetAccount(self): """ dont use account and retry download """ self.account = None @@ -208,7 +206,7 @@ class Hoster(Base): while self.pyfile.waitUntil > time(): self.thread.m.reconnecting.wait(2) - if self.pyfile.abort: raise Abort + self.checkAbort() if self.thread.m.reconnecting.isSet(): self.waiting = False self.wantReconnect = False @@ -243,88 +241,6 @@ class Hoster(Base): self.retries += 1 raise Retry(reason) - def invalidCaptcha(self): - if self.cTask: - self.cTask.invalid() - - def correctCaptcha(self): - if self.cTask: - self.cTask.correct() - - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual'): - """ Loads a captcha and decrypts it with ocr, plugin, user input - - :param url: url of captcha image - :param get: get part for request - :param post: post part for request - :param cookies: True if cookies should be enabled - :param forceUser: if True, ocr is not used - :param imgtype: Type of the Image - :param result_type: 'textual' if text is written on the captcha\ - or 'positional' for captcha where the user have to click\ - on a specific region on the captcha - - :return: result of decrypting - """ - - img = self.load(url, get=get, post=post, cookies=cookies) - - id = ("%.2f" % time())[-6:].replace(".", "") - temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - temp_file.write(img) - temp_file.close() - - has_plugin = self.__name__ in self.core.pluginManager.getPlugins("captcha") - - if self.core.captcha: - Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) - else: - Ocr = None - - if Ocr and not forceUser: - sleep(randint(3000, 5000) / 1000.0) - if self.pyfile.abort: raise Abort - - ocr = Ocr() - result = ocr.get_captcha(temp_file.name) - else: - captchaManager = self.core.captchaManager - task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) - self.cTask = task - captchaManager.handleCaptcha(task) - - while task.isWaiting(): - if self.pyfile.abort: - captchaManager.removeTask(task) - raise Abort - sleep(1) - - captchaManager.removeTask(task) - - if task.error and has_plugin: #ignore default error message since the user could use OCR - self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) - elif task.error: - self.fail(task.error) - elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) - - result = task.result - self.log.debug("Received captcha result: %s" % str(result)) - - if not self.core.debug: - try: - remove(temp_file.name) - except: - pass - - return result - - - def load(self, *args, **kwargs): - """ See 'Base' load method for more info """ - if self.pyfile.abort: raise Abort - return Base.load(self, *args, **kwargs) def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder @@ -338,8 +254,8 @@ class Hoster(Base): the filename will be changed if needed :return: The location where the file was saved """ - self.checkForSameFiles() + self.checkAbort() self.pyfile.setStatus("downloading") diff --git a/module/plugins/captcha/GigasizeCom.py b/module/plugins/captcha/GigasizeCom.py deleted file mode 100644 index d31742eb5..000000000 --- a/module/plugins/captcha/GigasizeCom.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -from captcha import OCR - -class GigasizeCom(OCR): - def __init__(self): - OCR.__init__(self) - - def get_captcha(self, image): - self.load_image(image) - self.threshold(2.8) - self.run_tesser(True, False, False, True) - return self.result_captcha - -if __name__ == '__main__': - ocr = GigasizeCom() - import urllib - urllib.urlretrieve('http://www.gigasize.com/randomImage.php', "gigasize_tmp.jpg") - - print ocr.get_captcha('gigasize_tmp.jpg') diff --git a/module/plugins/captcha/LinksaveIn.py b/module/plugins/captcha/LinksaveIn.py deleted file mode 100644 index 3ad7b265a..000000000 --- a/module/plugins/captcha/LinksaveIn.py +++ /dev/null @@ -1,147 +0,0 @@ -from captcha import OCR -import Image -from os import sep -from os.path import dirname -from os.path import abspath -from glob import glob - - -class LinksaveIn(OCR): - __name__ = "LinksaveIn" - def __init__(self): - OCR.__init__(self) - self.data_dir = dirname(abspath(__file__)) + sep + "LinksaveIn" + sep - - def load_image(self, image): - im = Image.open(image) - frame_nr = 0 - - lut = im.resize((256, 1)) - lut.putdata(range(256)) - lut = list(lut.convert("RGB").getdata()) - - new = Image.new("RGB", im.size) - npix = new.load() - while True: - try: - im.seek(frame_nr) - except EOFError: - break - frame = im.copy() - pix = frame.load() - for x in range(frame.size[0]): - for y in range(frame.size[1]): - if lut[pix[x, y]] != (0,0,0): - npix[x, y] = lut[pix[x, y]] - frame_nr += 1 - new.save(self.data_dir+"unblacked.png") - self.image = new.copy() - self.pixels = self.image.load() - self.result_captcha = '' - - def get_bg(self): - stat = {} - cstat = {} - img = self.image.convert("P") - for bgpath in glob(self.data_dir+"bg/*.gif"): - stat[bgpath] = 0 - bg = Image.open(bgpath) - - bglut = bg.resize((256, 1)) - bglut.putdata(range(256)) - bglut = list(bglut.convert("RGB").getdata()) - - lut = img.resize((256, 1)) - lut.putdata(range(256)) - lut = list(lut.convert("RGB").getdata()) - - bgpix = bg.load() - pix = img.load() - for x in range(bg.size[0]): - for y in range(bg.size[1]): - rgb_bg = bglut[bgpix[x, y]] - rgb_c = lut[pix[x, y]] - try: - cstat[rgb_c] += 1 - except: - cstat[rgb_c] = 1 - if rgb_bg == rgb_c: - stat[bgpath] += 1 - max_p = 0 - bg = "" - for bgpath, value in stat.items(): - if max_p < value: - bg = bgpath - max_p = value - return bg - - def substract_bg(self, bgpath): - bg = Image.open(bgpath) - img = self.image.convert("P") - - bglut = bg.resize((256, 1)) - bglut.putdata(range(256)) - bglut = list(bglut.convert("RGB").getdata()) - - lut = img.resize((256, 1)) - lut.putdata(range(256)) - lut = list(lut.convert("RGB").getdata()) - - bgpix = bg.load() - pix = img.load() - orgpix = self.image.load() - for x in range(bg.size[0]): - for y in range(bg.size[1]): - rgb_bg = bglut[bgpix[x, y]] - rgb_c = lut[pix[x, y]] - if rgb_c == rgb_bg: - orgpix[x, y] = (255,255,255) - - def eval_black_white(self): - new = Image.new("RGB", (140, 75)) - pix = new.load() - orgpix = self.image.load() - thresh = 4 - for x in range(new.size[0]): - for y in range(new.size[1]): - rgb = orgpix[x, y] - r, g, b = rgb - pix[x, y] = (255,255,255) - if r > max(b, g)+thresh: - pix[x, y] = (0,0,0) - if g < min(r, b): - pix[x, y] = (0,0,0) - if g > max(r, b)+thresh: - pix[x, y] = (0,0,0) - if b > max(r, g)+thresh: - pix[x, y] = (0,0,0) - self.image = new - self.pixels = self.image.load() - - def get_captcha(self, image): - self.load_image(image) - bg = self.get_bg() - self.substract_bg(bg) - self.eval_black_white() - self.to_greyscale() - self.image.save(self.data_dir+"cleaned_pass1.png") - self.clean(4) - self.clean(4) - self.image.save(self.data_dir+"cleaned_pass2.png") - letters = self.split_captcha_letters() - final = "" - for n, letter in enumerate(letters): - self.image = letter - self.image.save(ocr.data_dir+"letter%d.png" % n) - self.run_tesser(True, True, False, False) - final += self.result_captcha - - return final - -if __name__ == '__main__': - import urllib - ocr = LinksaveIn() - testurl = "http://linksave.in/captcha/cap.php?hsh=2229185&code=ZzHdhl3UffV3lXTH5U4b7nShXj%2Bwma1vyoNBcbc6lcc%3D" - urllib.urlretrieve(testurl, ocr.data_dir+"captcha.gif") - - print ocr.get_captcha(ocr.data_dir+'captcha.gif') diff --git a/module/plugins/captcha/MegauploadCom.py b/module/plugins/captcha/MegauploadCom.py deleted file mode 100644 index 469ee4094..000000000 --- a/module/plugins/captcha/MegauploadCom.py +++ /dev/null @@ -1,14 +0,0 @@ -from captcha import OCR - -class MegauploadCom(OCR): - __name__ = "MegauploadCom" - def __init__(self): - OCR.__init__(self) - - def get_captcha(self, image): - self.load_image(image) - self.run_tesser(True, True, False, True) - return self.result_captcha - -if __name__ == '__main__': - ocr = MegauploadCom() diff --git a/module/plugins/captcha/NetloadIn.py b/module/plugins/captcha/NetloadIn.py deleted file mode 100644 index 7f2e6a8d1..000000000 --- a/module/plugins/captcha/NetloadIn.py +++ /dev/null @@ -1,24 +0,0 @@ -from captcha import OCR - -class NetloadIn(OCR): - __name__ = "NetloadIn" - def __init__(self): - OCR.__init__(self) - - def get_captcha(self, image): - self.load_image(image) - self.to_greyscale() - self.clean(3) - self.clean(3) - self.run_tesser(True, True, False, False) - - self.result_captcha = self.result_captcha.replace(" ", "")[:4] # cut to 4 numbers - - return self.result_captcha - -if __name__ == '__main__': - import urllib - ocr = NetloadIn() - urllib.urlretrieve("http://netload.in/share/includes/captcha.php", "captcha.png") - - print ocr.get_captcha('captcha.png') diff --git a/module/plugins/captcha/ShareonlineBiz.py b/module/plugins/captcha/ShareonlineBiz.py deleted file mode 100644 index b07fb9b0f..000000000 --- a/module/plugins/captcha/ShareonlineBiz.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2009 kingzero, RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -### -from captcha import OCR - -class ShareonlineBiz(OCR): - __name__ = "ShareonlineBiz" - - def __init__(self): - OCR.__init__(self) - - def get_captcha(self, image): - self.load_image(image) - self.to_greyscale() - self.image = self.image.resize((160, 50)) - self.pixels = self.image.load() - self.threshold(1.85) - #self.eval_black_white(240) - #self.derotate_by_average() - - letters = self.split_captcha_letters() - - final = "" - for letter in letters: - self.image = letter - self.run_tesser(True, True, False, False) - final += self.result_captcha - - return final - - #tesseract at 60% - -if __name__ == '__main__': - import urllib - ocr = ShareonlineBiz() - urllib.urlretrieve("http://www.share-online.biz/captcha.php", "captcha.jpeg") - print ocr.get_captcha('captcha.jpeg') diff --git a/module/plugins/captcha/__init__.py b/module/plugins/captcha/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/module/plugins/captcha/captcha.py b/module/plugins/captcha/captcha.py deleted file mode 100644 index 4cbb736c1..000000000 --- a/module/plugins/captcha/captcha.py +++ /dev/null @@ -1,315 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2009 kingzero, RaNaN -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -### -from __future__ import with_statement -import os -from os.path import join -from os.path import abspath -import logging -import subprocess -#import tempfile - -import Image -import TiffImagePlugin -import PngImagePlugin -import GifImagePlugin -import JpegImagePlugin - - -class OCR(object): - - __name__ = "OCR" - - def __init__(self): - self.logger = logging.getLogger("log") - - def load_image(self, image): - self.image = Image.open(image) - self.pixels = self.image.load() - self.result_captcha = '' - - def unload(self): - """delete all tmp images""" - pass - - def threshold(self, value): - self.image = self.image.point(lambda a: a * value + 10) - - def run(self, command): - """Run a command""" - - popen = subprocess.Popen(command, bufsize = -1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - popen.wait() - output = popen.stdout.read() +" | "+ popen.stderr.read() - popen.stdout.close() - popen.stderr.close() - self.logger.debug("Tesseract ReturnCode %s Output: %s" % (popen.returncode, output)) - - def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True): - #self.logger.debug("create tmp tif") - - - #tmp = tempfile.NamedTemporaryFile(suffix=".tif") - tmp = open(join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") - tmp.close() - #self.logger.debug("create tmp txt") - #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") - tmpTxt = open(join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") - tmpTxt.close() - - self.logger.debug("save tiff") - self.image.save(tmp.name, 'TIFF') - - if os.name == "nt": - tessparams = [join(pypath,"tesseract","tesseract.exe")] - else: - tessparams = ['tesseract'] - - tessparams.extend( [abspath(tmp.name), abspath(tmpTxt.name).replace(".txt", "")] ) - - if subset and (digits or lowercase or uppercase): - #self.logger.debug("create temp subset config") - #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") - tmpSub = open(join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") - tmpSub.write("tessedit_char_whitelist ") - if digits: - tmpSub.write("0123456789") - if lowercase: - tmpSub.write("abcdefghijklmnopqrstuvwxyz") - if uppercase: - tmpSub.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ") - tmpSub.write("\n") - tessparams.append("nobatch") - tessparams.append(abspath(tmpSub.name)) - tmpSub.close() - - self.logger.debug("run tesseract") - self.run(tessparams) - self.logger.debug("read txt") - - try: - with open(tmpTxt.name, 'r') as f: - self.result_captcha = f.read().replace("\n", "") - except: - self.result_captcha = "" - - self.logger.debug(self.result_captcha) - try: - os.remove(tmp.name) - os.remove(tmpTxt.name) - if subset and (digits or lowercase or uppercase): - os.remove(tmpSub.name) - except: - pass - - def get_captcha(self, name): - raise NotImplementedError - - def to_greyscale(self): - if self.image.mode != 'L': - self.image = self.image.convert('L') - - self.pixels = self.image.load() - - def eval_black_white(self, limit): - self.pixels = self.image.load() - w, h = self.image.size - for x in xrange(w): - for y in xrange(h): - if self.pixels[x, y] > limit: - self.pixels[x, y] = 255 - else: - self.pixels[x, y] = 0 - - def clean(self, allowed): - pixels = self.pixels - - w, h = self.image.size - - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 255: continue - # no point in processing white pixels since we only want to remove black pixel - count = 0 - - try: - if pixels[x-1, y-1] != 255: count += 1 - if pixels[x-1, y] != 255: count += 1 - if pixels[x-1, y + 1] != 255: count += 1 - if pixels[x, y + 1] != 255: count += 1 - if pixels[x + 1, y + 1] != 255: count += 1 - if pixels[x + 1, y] != 255: count += 1 - if pixels[x + 1, y-1] != 255: count += 1 - if pixels[x, y-1] != 255: count += 1 - except: - pass - - # not enough neighbors are dark pixels so mark this pixel - # to be changed to white - if count < allowed: - pixels[x, y] = 1 - - # second pass: this time set all 1's to 255 (white) - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 1: pixels[x, y] = 255 - - self.pixels = pixels - - def derotate_by_average(self): - """rotate by checking each angle and guess most suitable""" - - w, h = self.image.size - pixels = self.pixels - - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 0: - pixels[x, y] = 155 - - highest = {} - counts = {} - - for angle in range(-45, 45): - - tmpimage = self.image.rotate(angle) - - pixels = tmpimage.load() - - w, h = self.image.size - - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 0: - pixels[x, y] = 255 - - - count = {} - - for x in xrange(w): - count[x] = 0 - for y in xrange(h): - if pixels[x, y] == 155: - count[x] += 1 - - sum = 0 - cnt = 0 - - for x in count.values(): - if x != 0: - sum += x - cnt += 1 - - avg = sum / cnt - counts[angle] = cnt - highest[angle] = 0 - for x in count.values(): - if x > highest[angle]: - highest[angle] = x - - highest[angle] = highest[angle] - avg - - hkey = 0 - hvalue = 0 - - for key, value in highest.iteritems(): - if value > hvalue: - hkey = key - hvalue = value - - self.image = self.image.rotate(hkey) - pixels = self.image.load() - - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 0: - pixels[x, y] = 255 - - if pixels[x, y] == 155: - pixels[x, y] = 0 - - self.pixels = pixels - - def split_captcha_letters(self): - captcha = self.image - started = False - letters = [] - width, height = captcha.size - bottomY, topY = 0, height - pixels = captcha.load() - - for x in xrange(width): - black_pixel_in_col = False - for y in xrange(height): - if pixels[x, y] != 255: - if not started: - started = True - firstX = x - lastX = x - - if y > bottomY: bottomY = y - if y < topY: topY = y - if x > lastX: lastX = x - - black_pixel_in_col = True - - if black_pixel_in_col == False and started == True: - rect = (firstX, topY, lastX, bottomY) - new_captcha = captcha.crop(rect) - - w, h = new_captcha.size - if w > 5 and h > 5: - letters.append(new_captcha) - - started = False - bottomY, topY = 0, height - - return letters - - def correct(self, values, var=None): - - if var: - result = var - else: - result = self.result_captcha - - for key, item in values.iteritems(): - - if key.__class__ == str: - result = result.replace(key, item) - else: - for expr in key: - result = result.replace(expr, item) - - if var: - return result - else: - self.result_captcha = result - - -if __name__ == '__main__': - ocr = OCR() - ocr.load_image("B.jpg") - ocr.to_greyscale() - ocr.eval_black_white(140) - ocr.derotate_by_average() - ocr.run_tesser() - print "Tesseract", ocr.result_captcha - ocr.image.save("derotated.jpg") - diff --git a/module/plugins/internal/NetloadInOCR.py b/module/plugins/internal/NetloadInOCR.py new file mode 100644 index 000000000..e50978701 --- /dev/null +++ b/module/plugins/internal/NetloadInOCR.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from OCR import OCR + +class NetloadInOCR(OCR): + __version__ = 0.1 + + def __init__(self): + OCR.__init__(self) + + def get_captcha(self, image): + self.load_image(image) + self.to_greyscale() + self.clean(3) + self.clean(3) + self.run_tesser(True, True, False, False) + + self.result_captcha = self.result_captcha.replace(" ", "")[:4] # cut to 4 numbers + + return self.result_captcha + +if __name__ == '__main__': + import urllib + ocr = NetloadInOCR() + urllib.urlretrieve("http://netload.in/share/includes/captcha.php", "captcha.png") + + print ocr.get_captcha('captcha.png') diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py new file mode 100644 index 000000000..9f8b7ef8c --- /dev/null +++ b/module/plugins/internal/OCR.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2009 kingzero, RaNaN +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +### +from __future__ import with_statement +import os +from os.path import join +from os.path import abspath +import logging +import subprocess +#import tempfile + +import Image +import TiffImagePlugin +import PngImagePlugin +import GifImagePlugin +import JpegImagePlugin + + +class OCR(object): + __version__ = 0.1 + + def __init__(self): + self.logger = logging.getLogger("log") + + def load_image(self, image): + self.image = Image.open(image) + self.pixels = self.image.load() + self.result_captcha = '' + + def unload(self): + """delete all tmp images""" + pass + + def threshold(self, value): + self.image = self.image.point(lambda a: a * value + 10) + + def run(self, command): + """Run a command""" + + popen = subprocess.Popen(command, bufsize = -1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + popen.wait() + output = popen.stdout.read() +" | "+ popen.stderr.read() + popen.stdout.close() + popen.stderr.close() + self.logger.debug("Tesseract ReturnCode %s Output: %s" % (popen.returncode, output)) + + def run_tesser(self, subset=False, digits=True, lowercase=True, uppercase=True): + #self.logger.debug("create tmp tif") + + + #tmp = tempfile.NamedTemporaryFile(suffix=".tif") + tmp = open(join("tmp", "tmpTif_%s.tif" % self.__name__), "wb") + tmp.close() + #self.logger.debug("create tmp txt") + #tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") + tmpTxt = open(join("tmp", "tmpTxt_%s.txt" % self.__name__), "wb") + tmpTxt.close() + + self.logger.debug("save tiff") + self.image.save(tmp.name, 'TIFF') + + if os.name == "nt": + tessparams = [join(pypath,"tesseract","tesseract.exe")] + else: + tessparams = ['tesseract'] + + tessparams.extend( [abspath(tmp.name), abspath(tmpTxt.name).replace(".txt", "")] ) + + if subset and (digits or lowercase or uppercase): + #self.logger.debug("create temp subset config") + #tmpSub = tempfile.NamedTemporaryFile(suffix=".subset") + tmpSub = open(join("tmp", "tmpSub_%s.subset" % self.__name__), "wb") + tmpSub.write("tessedit_char_whitelist ") + if digits: + tmpSub.write("0123456789") + if lowercase: + tmpSub.write("abcdefghijklmnopqrstuvwxyz") + if uppercase: + tmpSub.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + tmpSub.write("\n") + tessparams.append("nobatch") + tessparams.append(abspath(tmpSub.name)) + tmpSub.close() + + self.logger.debug("run tesseract") + self.run(tessparams) + self.logger.debug("read txt") + + try: + with open(tmpTxt.name, 'r') as f: + self.result_captcha = f.read().replace("\n", "") + except: + self.result_captcha = "" + + self.logger.debug(self.result_captcha) + try: + os.remove(tmp.name) + os.remove(tmpTxt.name) + if subset and (digits or lowercase or uppercase): + os.remove(tmpSub.name) + except: + pass + + def get_captcha(self, name): + raise NotImplementedError + + def to_greyscale(self): + if self.image.mode != 'L': + self.image = self.image.convert('L') + + self.pixels = self.image.load() + + def eval_black_white(self, limit): + self.pixels = self.image.load() + w, h = self.image.size + for x in xrange(w): + for y in xrange(h): + if self.pixels[x, y] > limit: + self.pixels[x, y] = 255 + else: + self.pixels[x, y] = 0 + + def clean(self, allowed): + pixels = self.pixels + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 255: continue + # no point in processing white pixels since we only want to remove black pixel + count = 0 + + try: + if pixels[x-1, y-1] != 255: count += 1 + if pixels[x-1, y] != 255: count += 1 + if pixels[x-1, y + 1] != 255: count += 1 + if pixels[x, y + 1] != 255: count += 1 + if pixels[x + 1, y + 1] != 255: count += 1 + if pixels[x + 1, y] != 255: count += 1 + if pixels[x + 1, y-1] != 255: count += 1 + if pixels[x, y-1] != 255: count += 1 + except: + pass + + # not enough neighbors are dark pixels so mark this pixel + # to be changed to white + if count < allowed: + pixels[x, y] = 1 + + # second pass: this time set all 1's to 255 (white) + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 1: pixels[x, y] = 255 + + self.pixels = pixels + + def derotate_by_average(self): + """rotate by checking each angle and guess most suitable""" + + w, h = self.image.size + pixels = self.pixels + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 155 + + highest = {} + counts = {} + + for angle in range(-45, 45): + + tmpimage = self.image.rotate(angle) + + pixels = tmpimage.load() + + w, h = self.image.size + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + + count = {} + + for x in xrange(w): + count[x] = 0 + for y in xrange(h): + if pixels[x, y] == 155: + count[x] += 1 + + sum = 0 + cnt = 0 + + for x in count.values(): + if x != 0: + sum += x + cnt += 1 + + avg = sum / cnt + counts[angle] = cnt + highest[angle] = 0 + for x in count.values(): + if x > highest[angle]: + highest[angle] = x + + highest[angle] = highest[angle] - avg + + hkey = 0 + hvalue = 0 + + for key, value in highest.iteritems(): + if value > hvalue: + hkey = key + hvalue = value + + self.image = self.image.rotate(hkey) + pixels = self.image.load() + + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 + + if pixels[x, y] == 155: + pixels[x, y] = 0 + + self.pixels = pixels + + def split_captcha_letters(self): + captcha = self.image + started = False + letters = [] + width, height = captcha.size + bottomY, topY = 0, height + pixels = captcha.load() + + for x in xrange(width): + black_pixel_in_col = False + for y in xrange(height): + if pixels[x, y] != 255: + if not started: + started = True + firstX = x + lastX = x + + if y > bottomY: bottomY = y + if y < topY: topY = y + if x > lastX: lastX = x + + black_pixel_in_col = True + + if black_pixel_in_col == False and started == True: + rect = (firstX, topY, lastX, bottomY) + new_captcha = captcha.crop(rect) + + w, h = new_captcha.size + if w > 5 and h > 5: + letters.append(new_captcha) + + started = False + bottomY, topY = 0, height + + return letters + + def correct(self, values, var=None): + + if var: + result = var + else: + result = self.result_captcha + + for key, item in values.iteritems(): + + if key.__class__ == str: + result = result.replace(key, item) + else: + for expr in key: + result = result.replace(expr, item) + + if var: + return result + else: + self.result_captcha = result + + +if __name__ == '__main__': + ocr = OCR() + ocr.load_image("B.jpg") + ocr.to_greyscale() + ocr.eval_black_white(140) + ocr.derotate_by_average() + ocr.run_tesser() + print "Tesseract", ocr.result_captcha + ocr.image.save("derotated.jpg") + diff --git a/module/plugins/internal/ShareonlineBizOCR.py b/module/plugins/internal/ShareonlineBizOCR.py new file mode 100644 index 000000000..c5c2e92e8 --- /dev/null +++ b/module/plugins/internal/ShareonlineBizOCR.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2009 kingzero, RaNaN +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +### +from OCR import OCR + +class ShareonlineBizOCR(OCR): + __version__ = 0.1 + + def __init__(self): + OCR.__init__(self) + + def get_captcha(self, image): + self.load_image(image) + self.to_greyscale() + self.image = self.image.resize((160, 50)) + self.pixels = self.image.load() + self.threshold(1.85) + #self.eval_black_white(240) + #self.derotate_by_average() + + letters = self.split_captcha_letters() + + final = "" + for letter in letters: + self.image = letter + self.run_tesser(True, True, False, False) + final += self.result_captcha + + return final + + #tesseract at 60% + +if __name__ == '__main__': + import urllib + ocr = ShareonlineBizOCR() + urllib.urlretrieve("http://www.share-online.biz/captcha.php", "captcha.jpeg") + print ocr.get_captcha('captcha.jpeg') -- cgit v1.2.3 From b40b32ee05f611323a7827fad2a25fa0a28dcb24 Mon Sep 17 00:00:00 2001 From: X3n0m0rph59 Date: Sun, 22 Apr 2012 19:56:17 +0200 Subject: a huge pile of spelling fixes --- module/plugins/Account.py | 32 ++++++++++++++-------------- module/plugins/Addon.py | 8 +++---- module/plugins/Base.py | 10 ++++----- module/plugins/Crypter.py | 14 ++++++------ module/plugins/Hoster.py | 14 ++++++------ module/plugins/PluginManager.py | 4 ++-- module/plugins/addons/Ev0InFetcher.py | 4 ++-- module/plugins/hoster/NetloadIn.py | 4 ++-- module/plugins/internal/AbstractExtractor.py | 12 +++++------ module/plugins/internal/SimpleHoster.py | 2 +- 10 files changed, 52 insertions(+), 52 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Account.py b/module/plugins/Account.py index 28d1387fd..7c24298e7 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -16,10 +16,10 @@ class WrongPassword(Exception): #noinspection PyUnresolvedReferences class Account(Base, AccountInfo): """ - Base class for every Account plugin. - Just overwrite `login` and cookies will be stored and account becomes accessible in\ + Base class for every account plugin. + Just overwrite `login` and cookies will be stored and the account becomes accessible in\ associated hoster plugin. Plugin should also provide `loadAccountInfo`. \ - A instance of this class is created for every entered account, it holds all \ + An instance of this class is created for every entered account, it holds all \ fields of AccountInfo ttype, and can be set easily at runtime. """ @@ -78,7 +78,7 @@ class Account(Base, AccountInfo): pass def login(self, req): - """login into account, the cookies will be saved so user can be recognized + """login into account, the cookies will be saved so the user can be recognized :param req: `Request` instance """ @@ -101,7 +101,7 @@ class Account(Base, AccountInfo): try: self.login(req) except TypeError: #TODO: temporary - self.logDebug("Deprecated .login(...) signature ommit user, data") + self.logDebug("Deprecated .login(...) signature omit user, data") self.login(self.loginname, {"password": self.password}, req) @@ -129,10 +129,10 @@ class Account(Base, AccountInfo): self.premium = Account.premium def update(self, password=None, options=None): - """ updates account and return true if anything changed """ + """ updates the account and returns true if anything changed """ self.login_ts = 0 - self.valid = True #set valid so it will be retried to login + self.valid = True #set valid, so the login will be retried if "activated" in options: self.activated = from_string(options["avtivated"], "bool") @@ -163,8 +163,8 @@ class Account(Base, AccountInfo): @lock def getAccountInfo(self, force=False): - """retrieve account infos for an user, do **not** overwrite this method!\\ - just use it to retrieve infos in hoster plugins. see `loadAccountInfo` + """retrieve account info's for an user, do **not** overwrite this method!\\ + just use it to retrieve info's in hoster plugins. see `loadAccountInfo` :param name: username :param force: reloads cached account information @@ -180,7 +180,7 @@ class Account(Base, AccountInfo): try: infos = self.loadAccountInfo(req) except TypeError: #TODO: temporary - self.logDebug("Deprecated .loadAccountInfo(...) signature, ommit user argument.") + self.logDebug("Deprecated .loadAccountInfo(...) signature, omit user argument.") infos = self.loadAccountInfo(self.loginname, req) except Exception, e: infos = {"error": str(e)} @@ -221,7 +221,7 @@ class Account(Base, AccountInfo): return self.premium def isUsable(self): - """Check several contraints to determine if account should be used""" + """Check several constraints to determine if account should be used""" if not self.valid or not self.activated: return False if self.options["time"]: @@ -232,11 +232,11 @@ class Account(Base, AccountInfo): if not compare_time(start.split(":"), end.split(":")): return False except: - self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + self.logWarning(_("Your Time %s has a wrong format, use: 1:22-3:44") % time_data) if 0 <= self.validuntil < time(): return False - if self.trafficleft is 0: # test explicity for 0 + if self.trafficleft is 0: # test explicitly for 0 return False return True @@ -269,15 +269,15 @@ class Account(Base, AccountInfo): self.scheduleRefresh(60 * 60) def scheduleRefresh(self, time=0, force=True): - """ add task to refresh account info to sheduler """ + """ add a task for refreshing the account info to the scheduler """ self.logDebug("Scheduled Account refresh for %s in %s seconds." % (self.loginname, time)) self.core.scheduler.addJob(time, self.getAccountInfo, [force]) @lock def checkLogin(self, req): - """ checks if user is still logged in """ + """ checks if the user is still logged in """ if self.login_ts + self.login_timeout * 60 < time(): - if self.login_ts: # seperate from fresh login to have better debug logs + if self.login_ts: # separate from fresh login to have better debug logs self.logDebug("Reached login timeout for %s" % self.loginname) else: self.logDebug("Login with %s" % self.loginname) diff --git a/module/plugins/Addon.py b/module/plugins/Addon.py index fe9ae4817..3fc4eb467 100644 --- a/module/plugins/Addon.py +++ b/module/plugins/Addon.py @@ -81,20 +81,20 @@ def threaded(f): class Addon(Base): """ - Base class for addon plugins. Use @threaded decorator for all longer running task. + Base class for addon plugins. Use @threaded decorator for all longer running tasks. - Decorate methods with @Expose, @AddventListener, @ConfigHandler + Decorate methods with @Expose, @AddEventListener, @ConfigHandler """ - #: automatically register event listeners for functions, attribute will be deleted dont use it yourself + #: automatically register event listeners for functions, attribute will be deleted don't use it yourself event_map = None # Alternative to event_map #: List of events the plugin can handle, name the functions exactly like eventname. event_list = None # dont make duplicate entries in event_map - #: periodic call interval in secondc + #: periodic call interval in seconds interval = 60 def __init__(self, core, manager): diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 61fa211f4..4649a2b08 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -95,8 +95,8 @@ class Base(object): def logInfo(self, *args, **kwargs): """ Print args to log at specific level - :param args: Arbitary object which should be logged - :param kwargs: sep=(how to seperate arguments), default = " | " + :param args: Arbitrary object which should be logged + :param kwargs: sep=(how to separate arguments), default = " | " """ self._log("info", *args, **kwargs) @@ -173,7 +173,7 @@ class Base(object): return False def checkAbort(self): - """ Will be overwriten to determine if control flow should be aborted """ + """ Will be overwritten to determine if control flow should be aborted """ if self.abort: raise Abort() def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): @@ -185,7 +185,7 @@ class Base(object): :param ref: Set HTTP_REFERER header :param cookies: use saved cookies :param just_header: if True only the header will be retrieved and returned as dict - :param decode: Wether to decode the output according to http header, should be True in most cases + :param decode: Whether to decode the output according to http header, should be True in most cases :return: Loaded content """ if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") @@ -308,7 +308,7 @@ class Base(object): elif task.error: self.fail(task.error) elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) + self.fail(_("No captcha result obtained in appropriate time by any of the plugins.")) result = task.result self.log.debug("Received captcha result: %s" % str(result)) diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index 15feea8e0..920009f44 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -9,7 +9,7 @@ from module.utils.fs import exists, remove, fs_encode from Base import Base, Retry class Package: - """ Container that indicates new package should be created """ + """ Container that indicates that a new package should be created """ def __init__(self, name, urls=None): self.name = name self.urls = urls if urls else [] @@ -102,14 +102,14 @@ class Crypter(Base): Base.__init__(self, core) self.req = core.requestFactory.getRequest(self.__name__) - # Package the plugin was initialized for, dont use this, its not guaranteed to be set + # Package the plugin was initialized for, don't use this, its not guaranteed to be set self.package = package #: Password supplied by user self.password = password #: Propose a renaming of the owner package self.rename = None - # For old style decrypter, do not use these ! + # For old style decrypter, do not use these! self.packages = [] self.urls = [] self.pyfile = None @@ -120,7 +120,7 @@ class Crypter(Base): """More init stuff if needed""" def setup(self): - """Called everytime before decrypting. A Crypter plugin will be most likly used for several jobs.""" + """Called everytime before decrypting. A Crypter plugin will be most likely used for several jobs.""" def decryptURL(self, url): """Decrypt a single url @@ -150,7 +150,7 @@ class Crypter(Base): raise NotImplementedError def generatePackages(self, urls): - """Generates :class:`Package` instances and names from urls. Usefull for many different links and no\ + """Generates :class:`Package` instances and names from urls. Useful for many different links and no\ given package name. :param urls: list of urls @@ -166,7 +166,7 @@ class Crypter(Base): """ cls = self.__class__ - # seperate local and remote files + # separate local and remote files content, urls = self.getLocalContent(urls) if has_method(cls, "decryptURLs"): @@ -214,7 +214,7 @@ class Crypter(Base): return [] def getLocalContent(self, urls): - """Load files from disk and seperate to file content and url list + """Load files from disk and separate to file content and url list :param urls: :return: list of (filename, content), remote urls diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index b330743e6..737bdcdb4 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -48,7 +48,7 @@ class Hoster(Base): def getInfo(urls): """This method is used to retrieve the online status of files for hoster plugins. It has to *yield* list of tuples with the result in this format (name, size, status, url), - where status is one of API pyfile statusses. + where status is one of API pyfile statuses. :param urls: List of urls :return: yield list of tuple with results (name, size, status, url) @@ -108,11 +108,11 @@ class Hoster(Base): self.init() def getMultiDL(self): - self.logDebug("Deprectated attribute multiDL, use limitDL instead") + self.logDebug("Deprecated attribute multiDL, use limitDL instead") return self.limitDL <= 0 def setMultiDL(self, val): - self.logDebug("Deprectated attribute multiDL, use limitDL instead") + self.logDebug("Deprecated attribute multiDL, use limitDL instead") self.limitDL = 0 if val else 1 multiDL = property(getMultiDL, setMultiDL) @@ -142,7 +142,7 @@ class Hoster(Base): pass def setup(self): - """ setup for enviroment and other things, called before downloading (possibly more than one time)""" + """ setup for environment and other things, called before downloading (possibly more than one time)""" pass def preprocessing(self, thread): @@ -150,7 +150,7 @@ class Hoster(Base): self.thread = thread if self.account: - # will force a relogin or reload of account info if necessary + # will force a re-login or reload of account info if necessary self.account.getAccountInfo() else: self.req.clearCookies() @@ -169,7 +169,7 @@ class Hoster(Base): return self.pyfile.abort def resetAccount(self): - """ dont use account and retry download """ + """ don't use account and retry download """ self.account = None self.req = self.core.requestFactory.getRequest(self.__name__) self.retry() @@ -372,7 +372,7 @@ class Hoster(Base): if pyfile.status in (0, 12): #finished or downloading raise SkipDownload(pyfile.pluginname) elif pyfile.status in ( - 5, 7) and starting: #a download is waiting/starting and was appenrently started before + 5, 7) and starting: #a download is waiting/starting and was apparently started before raise SkipDownload(pyfile.pluginname) download_folder = self.config['general']['download_folder'] diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index 733cd2c5d..f42bd08c6 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -221,7 +221,7 @@ class PluginManager: def parseUrls(self, urls): - """parse plugins for given list of urls, seperate to crypter and hoster""" + """parse plugins for given list of urls, separate to crypter and hoster""" res = {"hoster": [], "crypter": []} # tupels of (url, plugin) @@ -313,7 +313,7 @@ class PluginManager: def find_module(self, fullname, path=None): #redirecting imports if necesarry - if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins + if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #separate pyload plugins if fullname.startswith(self.USERROOT): user = 1 else: user = 0 #used as bool and int diff --git a/module/plugins/addons/Ev0InFetcher.py b/module/plugins/addons/Ev0InFetcher.py index aeb46320a..608baf217 100644 --- a/module/plugins/addons/Ev0InFetcher.py +++ b/module/plugins/addons/Ev0InFetcher.py @@ -27,9 +27,9 @@ class Ev0InFetcher(Addon): __config__ = [("activated", "bool", "Activated", "False"), ("interval", "int", "Check interval in minutes", "10"), ("queue", "bool", "Move new shows directly to Queue", False), - ("shows", "str", "Shows to check for (comma seperated)", ""), + ("shows", "str", "Shows to check for (comma separated)", ""), ("quality", "xvid;x264;rmvb", "Video Format", "xvid"), - ("hoster", "str", "Hoster to use (comma seperated)", "NetloadIn,RapidshareCom,MegauploadCom,HotfileCom")] + ("hoster", "str", "Hoster to use (comma separated)", "NetloadIn,RapidshareCom,MegauploadCom,HotfileCom")] __author_name__ = ("mkaay") __author_mail__ = ("mkaay@mkaay.de") diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py index 382328496..d768090e8 100644 --- a/module/plugins/hoster/NetloadIn.py +++ b/module/plugins/hoster/NetloadIn.py @@ -10,7 +10,7 @@ from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL def getInfo(urls): - ## returns list of tupels (name, size (in bytes), status (see FileDatabase), url) + ## returns list of tuples (name, size (in bytes), status (see FileDatabase), url) apiurl = "http://api.netload.in/info.php?auth=Zf9SnQh9WiReEsb18akjvQGqT0I830e8&bz=1&md5=1&file_id=" @@ -196,7 +196,7 @@ class NetloadIn(Hoster): file_id = re.search('', page).group(1) if not captchawaited: wait = self.get_wait_time(page) - if i == 0: self.pyfile.waitUntil = time() # dont wait contrary to time on website + if i == 0: self.pyfile.waitUntil = time() # don't wait contrary to time on web site else: self.pyfile.waitUntil = t self.log.info(_("Netload: waiting for captcha %d s.") % (self.pyfile.waitUntil - time())) #self.setWait(wait) diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index ceb188193..3cd635eff 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -13,7 +13,7 @@ class WrongPassword(Exception): class AbtractExtractor: @staticmethod def checkDeps(): - """ Check if system statisfy dependencies + """ Check if system satisfies dependencies :return: boolean """ return True @@ -21,7 +21,7 @@ class AbtractExtractor: @staticmethod def getTargets(files_ids): """ Filter suited targets from list of filename id tuple list - :param files_ids: List of filepathes + :param files_ids: List of file paths :return: List of targets, id tuple list """ raise NotImplementedError @@ -30,10 +30,10 @@ class AbtractExtractor: def __init__(self, m, file, out, fullpath, overwrite, renice): """Initialize extractor for specific file - :param m: ExtractArchive Addon plugin - :param file: Absolute filepath + :param m: ExtractArchive addon plugin + :param file: Absolute file path :param out: Absolute path to destination directory - :param fullpath: extract to fullpath + :param fullpath: Extract to fullpath :param overwrite: Overwrite existing archives :param renice: Renice value """ @@ -52,7 +52,7 @@ class AbtractExtractor: def checkArchive(self): - """Check if password if needed. Raise ArchiveError if integrity is + """Check if password is needed. Raise ArchiveError if integrity is questionable. :return: boolean diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 69909a8a1..20263064a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -103,7 +103,7 @@ class SimpleHoster(Hoster): or FILE_NAME_INFO = r'(?Pfile_name)' and FILE_SIZE_INFO = r'(?Pfile_size) (?Punits)' FILE_OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN = r'Server maintainance' + TEMP_OFFLINE_PATTERN = r'Server maintenance' """ FILE_SIZE_REPLACEMENTS = [] -- cgit v1.2.3 From e8eaa91da9e1236d8009df0d79bebe023de8933f Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 6 May 2012 13:04:15 +0200 Subject: little documentation update --- module/plugins/Base.py | 2 +- module/plugins/Hoster.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 4649a2b08..905744c9e 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -300,7 +300,7 @@ class Base(object): raise Abort() sleep(1) - #TODO + #TODO task handling self.im.removeTask(task) if task.error and has_plugin: #ignore default error message since the user could use OCR diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 737bdcdb4..ad4f8f16b 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -100,7 +100,6 @@ class Hoster(Base): self.lastCheck = None #: js engine, see `JsEngine` self.js = self.core.js - self.cTask = None #captcha task self.retries = 0 # amount of retries already made self.html = None # some plugins store html code here @@ -108,13 +107,12 @@ class Hoster(Base): self.init() def getMultiDL(self): - self.logDebug("Deprecated attribute multiDL, use limitDL instead") return self.limitDL <= 0 def setMultiDL(self, val): - self.logDebug("Deprecated attribute multiDL, use limitDL instead") self.limitDL = 0 if val else 1 + #: virtual attribute using self.limitDL on behind multiDL = property(getMultiDL, setMultiDL) def getChunkCount(self): -- cgit v1.2.3 From 3d7ed0c367cf521b61ec1f78784dec73a3b7c32a Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 6 May 2012 20:40:39 +0200 Subject: some classes for multi user mode --- module/plugins/UserAddon.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 module/plugins/UserAddon.py (limited to 'module/plugins') diff --git a/module/plugins/UserAddon.py b/module/plugins/UserAddon.py new file mode 100644 index 000000000..c49b1ef41 --- /dev/null +++ b/module/plugins/UserAddon.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from Addon import Addon + +class UserAddon(Addon): + """ + Special type of an addon that only works a specific user. Has a configuration for every user who added it . + """ \ No newline at end of file -- cgit v1.2.3 From 2a74f88fcaf3d3baac24397de81a967c0b8c73e2 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 17 May 2012 20:06:11 +0200 Subject: small typo fixes and TODOs --- module/plugins/Addon.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Addon.py b/module/plugins/Addon.py index 3fc4eb467..614c68c80 100644 --- a/module/plugins/Addon.py +++ b/module/plugins/Addon.py @@ -20,7 +20,7 @@ from traceback import print_exc #from functools import wraps -from module.utils import has_method +from module.utils import has_method, to_list from Base import Base @@ -34,10 +34,14 @@ class Expose(object): return f def AddEventListener(event): - """ Used to register method for events. Arguments needs to match parameter of event """ + """ Used to register method for events. Arguments needs to match parameter of event + + :param event: Name of event or list of them. + """ class _klass(object): def __new__(cls, f, *args, **kwargs): - addonManager.addEventListener(class_name(f.__module__), f.func_name, event) + for ev in to_list(event): + addonManager.addEventListener(class_name(f.__module__), f.func_name, ev) return f return _klass @@ -74,6 +78,8 @@ def AddonInfo(desc): pass def threaded(f): + """ Decorator to run method in a thread. """ + #@wraps(f) def run(*args,**kwargs): addonManager.startThread(f, *args, **kwargs) -- cgit v1.2.3 From 0d2d6daef850ac6bcc7fafccd230e52d2a862c2c Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 3 Jun 2012 17:45:10 +0200 Subject: updates for database + api --- module/plugins/AccountManager.py | 141 ------------ module/plugins/Addon.py | 14 +- module/plugins/PluginManager.py | 405 --------------------------------- module/plugins/hoster/MegauploadCom.py | 6 +- 4 files changed, 5 insertions(+), 561 deletions(-) delete mode 100644 module/plugins/AccountManager.py delete mode 100644 module/plugins/PluginManager.py (limited to 'module/plugins') diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py deleted file mode 100644 index c610d10e0..000000000 --- a/module/plugins/AccountManager.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from threading import Lock -from random import choice - -from module.common.json_layer import json -from module.utils import lock - -class AccountManager: - """manages all accounts""" - - def __init__(self, core): - """Constructor""" - - self.core = core - self.lock = Lock() - - self.loadAccounts() - - def loadAccounts(self): - """loads all accounts available""" - - self.accounts = {} - - for plugin, loginname, activated, password, options in self.core.db.loadAccounts(): - # put into options as used in other context - options = json.loads(options) if options else {} - options["activated"] = activated - - self.createAccount(plugin, loginname, password, options) - - return - - def iterAccounts(self): - """ yields login, account for all accounts""" - for name, data in self.accounts.iteritems(): - for login, account in data.iteritems(): - yield login, account - - def saveAccounts(self): - """save all account information""" - - data = [] - for name, plugin in self.accounts.iteritems(): - data.extend([(name, acc.loginname, acc.activated, acc.password, json.dumps(acc.options)) for acc in - plugin.itervalues()]) - self.core.db.saveAccounts(data) - - def createAccount(self, plugin, loginname, password, options): - klass = self.core.pluginManager.loadClass("accounts", plugin) - if not klass: - self.core.log.warning(_("Unknown account plugin %s") % plugin) - return - - if plugin not in self.accounts: - self.accounts[plugin] = {} - - self.core.log.debug("Create account %s:%s" % (plugin, loginname)) - - self.accounts[plugin][loginname] = klass(self, loginname, password, options) - - - def getAccount(self, plugin, user): - return self.accounts[plugin].get(user, None) - - @lock - def updateAccount(self, plugin, user, password=None, options={}): - """add or update account""" - if plugin in self.accounts and user in self.accounts[plugin]: - acc = self.accounts[plugin][user] - updated = acc.update(password, options) - - self.saveAccounts() - if updated: acc.scheduleRefresh(force=True) - else: - self.createAccount(plugin, user, password, options) - self.saveAccounts() - - self.sendChange(plugin, user) - - @lock - def removeAccount(self, plugin, user): - """remove account""" - if plugin in self.accounts and user in self.accounts[plugin]: - del self.accounts[plugin][user] - self.core.db.removeAccount(plugin, user) - self.core.eventManager.dispatchEvent("accountDeleted", plugin, user) - else: - self.core.log.debug("Remove non existing account %s %s" % (plugin, user)) - - - @lock - def getAccountForPlugin(self, plugin): - if plugin in self.accounts: - accs = [x for x in self.accounts[plugin].values() if x.isUsable()] - if accs: return choice(accs) - - return None - - @lock - def getAllAccounts(self, refresh=False): - """ Return account info, refresh afterwards if needed - - :param refresh: - :return: - """ - if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAllAccounts) - - # load unavailable account info - for p_dict in self.accounts.itervalues(): - for acc in p_dict.itervalues(): - acc.getAccountInfo() - - return self.accounts - - def refreshAllAccounts(self): - """ Force a refresh of every account """ - for p in self.accounts.itervalues(): - for acc in p.itervalues(): - acc.getAccountInfo(True) - - def sendChange(self, plugin, name): - self.core.eventManager.dispatchEvent("accountUpdated", plugin, name) \ No newline at end of file diff --git a/module/plugins/Addon.py b/module/plugins/Addon.py index 614c68c80..60223dd28 100644 --- a/module/plugins/Addon.py +++ b/module/plugins/Addon.py @@ -27,11 +27,6 @@ from Base import Base def class_name(p): return p.rpartition(".")[2] -class Expose(object): - """ Used for decoration to declare rpc services. You can use any arbitrary method """ - def __new__(cls, f, *args, **kwargs): - addonManager.addRPC(class_name(f.__module__), f.func_name, f.func_doc) - return f def AddEventListener(event): """ Used to register method for events. Arguments needs to match parameter of event @@ -59,13 +54,12 @@ class ConfigHandler(object): addonManager.addConfigHandler(class_name(f.__module__), f.func_name) return f -def FileHandler(desc, media, package=False): - """ Register Handler for Files or packages. - Depending on package=True the decorated method needs to accept pid or fid as argument +def AddonHandler(desc, media=None): + """ Register Handler for files, packages, or arbitrary callable methods. + To let the method work on packages/files, media must be set and the argument named pid or fid. :param desc: verbose description - :param media: media type for which your method will be used - :param package: True if it works on packages + :param media: if True or bits of media type """ pass diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py deleted file mode 100644 index f42bd08c6..000000000 --- a/module/plugins/PluginManager.py +++ /dev/null @@ -1,405 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay, RaNaN -""" - -import re -import sys - -from os import listdir, makedirs -from os.path import isfile, join, exists, abspath, basename -from sys import version_info -from time import time - -from module.lib.SafeEval import const_eval as literal_eval -from module.plugins.Base import Base - -from new_collections import namedtuple - -#TODO: ignores not updatable - -# ignore these plugin configs, mainly because plugins were wiped out -IGNORE = ( - "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('addons', 'UnRar'), - 'EasyShareCom', 'FlyshareCz' - ) - -PluginTuple = namedtuple("PluginTuple", "version re deps user path") - -class PluginManager: - ROOT = "module.plugins." - USERROOT = "userplugins." - TYPES = ("crypter", "hoster", "accounts", "addons", "internal") - - BUILTIN = re.compile(r'__(?P[a-z0-9_]+)__\s*=\s?(True|False|None|[0-9x.]+)', re.I) - SINGLE = re.compile(r'__(?P[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?[a-z0-9_]+)__\s*=\s*((?:\{|\[|"{3}).*?(?:"""|\}|\]))', re.DOTALL | re.M | re.I) - - def __init__(self, core): - self.core = core - - #self.config = self.core.config - self.log = core.log - - self.plugins = {} - self.modules = {} # cached modules - self.history = [] # match history to speedup parsing (type, name) - self.createIndex() - - self.core.config.parseValues(self.core.config.PLUGIN) - - #register for import addon - sys.meta_path.append(self) - - - def logDebug(self, type, plugin, msg): - self.log.debug("Plugin %s | %s: %s" % (type, plugin, msg)) - - def createIndex(self): - """create information for all plugins available""" - # add to path, so we can import from userplugins - sys.path.append(abspath("")) - - if not exists("userplugins"): - makedirs("userplugins") - if not exists(join("userplugins", "__init__.py")): - f = open(join("userplugins", "__init__.py"), "wb") - f.close() - - a = time() - for type in self.TYPES: - self.plugins[type] = self.parse(type) - - self.log.debug("Created index of plugins in %.2f ms", (time() - a) * 1000) - - def parse(self, folder, home=None): - """ Analyze and parses all plugins in folder """ - plugins = {} - if home: - pfolder = join("userplugins", folder) - if not exists(pfolder): - makedirs(pfolder) - if not exists(join(pfolder, "__init__.py")): - f = open(join(pfolder, "__init__.py"), "wb") - f.close() - - else: - pfolder = join(pypath, "module", "plugins", folder) - - for f in listdir(pfolder): - if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( - "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): - continue - elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): - continue - elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): - continue - - # replace suffix and version tag - name = f[:-3] - if name[-1] == ".": name = name[:-4] - - plugin = self.parsePlugin(join(pfolder, f), folder, name, home) - if plugin: - plugins[name] = plugin - - if not home: - temp = self.parse(folder, plugins) - plugins.update(temp) - - return plugins - - def parseAttributes(self, filename, name, folder=""): - """ Parse attribute dict from plugin""" - data = open(filename, "rb") - content = data.read() - data.close() - - attrs = {} - for m in self.BUILTIN.findall(content) + self.SINGLE.findall(content) + self.MULTI.findall(content): - #replace gettext function and eval result - try: - attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) - except: - self.logDebug(folder, name, "Error when parsing: %s" % m[-1]) - self.core.print_exc() - - if not hasattr(Base, "__%s__" % m[0]): - if m[0] != "type": #TODO remove type from all plugins, its not needed - self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) - - return attrs - - def parsePlugin(self, filename, folder, name, home=None): - """ Parses a plugin from disk, folder means plugin type in this context. Also sets config. - - :arg home: dict with plugins, of which the found one will be matched against (according version) - :returns PluginTuple""" - - attrs = self.parseAttributes(filename, name, folder) - if not attrs: return - - version = 0 - - if "version" in attrs: - try: - version = float(attrs["version"]) - except ValueError: - self.logDebug(folder, name, "Invalid version %s" % attrs["version"]) - version = 9 #TODO remove when plugins are fixed, causing update loops - else: - self.logDebug(folder, name, "No version attribute") - - # home contains plugins from pyload root - if home and name in home: - if home[name].version >= version: - return - - if name in IGNORE or (folder, name) in IGNORE: - return - - if "pattern" in attrs and attrs["pattern"]: - try: - plugin_re = re.compile(attrs["pattern"]) - except: - self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"]) - plugin_re = None - else: plugin_re = None - - deps = attrs.get("dependencies", None) - - # create plugin tuple - plugin = PluginTuple(version, plugin_re, deps, bool(home), filename) - - - # internals have no config - if folder == "internal": - return plugin - - if folder == "addons" and "config" not in attrs and not attrs.get("internal", False): - attrs["config"] = (["activated", "bool", "Activated", False],) - - if "config" in attrs and attrs["config"]: - config = attrs["config"] - desc = attrs.get("description", "") - long_desc = attrs.get("long_description", "") - - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] - - if folder == "addons" and not attrs.get("internal", False): - for item in config: - if item[0] == "activated": break - else: # activated flag missing - config.insert(0, ("activated", "bool", "Activated", False)) - - try: - self.core.config.addConfigSection(name, name, desc, long_desc, config) - except: - self.logDebug(folder, name, "Invalid config %s" % config) - - return plugin - - - def parseUrls(self, urls): - """parse plugins for given list of urls, separate to crypter and hoster""" - - res = {"hoster": [], "crypter": []} # tupels of (url, plugin) - - for url in urls: - if type(url) not in (str, unicode, buffer): - self.log.debug("Parsing invalid type %s" % type(url)) - continue - - found = False - - for ptype, name in self.history: - if self.plugins[ptype][name].re.match(url): - res[ptype].append((url, name)) - found = (ptype, name) - break # need to exit this loop first - - if found: # found match - if self.history[0] != found: #update history - self.history.remove(found) - self.history.insert(0, found) - continue - - for ptype in ("crypter", "hoster"): - for name, plugin in self.plugins[ptype].iteritems(): - if plugin.re.match(url): - res[ptype].append((url, name)) - self.history.insert(0, (ptype, name)) - del self.history[10:] # cut down to size of 10 - found = True - break - - if not found: - res["hoster"].append((url, "BasePlugin")) - - return res["hoster"], res["crypter"] - - def getPlugins(self, type): - return self.plugins.get(type, None) - - def findPlugin(self, name, pluginlist=("hoster", "crypter")): - for ptype in pluginlist: - if name in self.plugins[ptype]: - return ptype, self.plugins[ptype][name] - return None, None - - def getPluginModule(self, name): - """ Decprecated: return plugin module from hoster|crypter""" - self.log.debug("Deprecated method: .getPluginModule()") - type, plugin = self.findPlugin(name) - return self.loadModule(type, name) - - def getPluginClass(self, name): - """ return plugin class from hoster|crypter, always the not overwritten one """ - type, plugin = self.findPlugin(name) - return self.loadClass(type, name) - - # MultiHoster will overwrite this - getPlugin = getPluginClass - - - def loadAttributes(self, type, name): - plugin = self.plugins[type][name] - return self.parseAttributes(plugin.path, name, type) - - - def loadModule(self, type, name): - """ Returns loaded module for plugin - - :param type: plugin type, subfolder of module.plugins - :param name: - """ - plugins = self.plugins[type] - if name in plugins: - if (type, name) in self.modules: return self.modules[(type, name)] - try: - # convert path to python recognizable import - path = basename(plugins[name].path).replace(".pyc", "").replace(".py", "") - module = __import__(self.ROOT + "%s.%s" % (type, path), globals(), locals(), path) - self.modules[(type, name)] = module # cache import, maybe unneeded - return module - except Exception, e: - self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) - self.core.print_exc() - - def loadClass(self, type, name): - """Returns the class of a plugin with the same name""" - module = self.loadModule(type, name) - if module: return getattr(module, name) - - def find_module(self, fullname, path=None): - #redirecting imports if necesarry - if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #separate pyload plugins - if fullname.startswith(self.USERROOT): user = 1 - else: user = 0 #used as bool and int - - split = fullname.split(".") - if len(split) != 4 - user: return - type, name = split[2 - user:4 - user] - - if type in self.plugins and name in self.plugins[type]: - #userplugin is a newer version - if not user and self.plugins[type][name].user: - return self - #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name].user: - return self - - - def load_module(self, name, replace=True): - if name not in sys.modules: #could be already in modules - if replace: - if self.ROOT in name: - newname = name.replace(self.ROOT, self.USERROOT) - else: - newname = name.replace(self.USERROOT, self.ROOT) - else: newname = name - - base, plugin = newname.rsplit(".", 1) - - self.log.debug("Redirected import %s -> %s" % (name, newname)) - - module = __import__(newname, globals(), locals(), [plugin]) - #inject under new an old name - sys.modules[name] = module - sys.modules[newname] = module - - return sys.modules[name] - - - def reloadPlugins(self, type_plugins): - """ reloads and reindexes plugins """ - if not type_plugins: return False - - self.log.debug("Request reload of plugins: %s" % type_plugins) - - as_dict = {} - for t, n in type_plugins: - if t in as_dict: - as_dict[t].append(n) - else: - as_dict[t] = [n] - - # we do not reload addons or internals, would cause to much side effects - if "addons" in as_dict or "internal" in as_dict: - return False - - for type in as_dict.iterkeys(): - for plugin in as_dict[type]: - if plugin in self.plugins[type]: - if (type, plugin) in self.modules: - self.log.debug("Reloading %s" % plugin) - reload(self.modules[(type, plugin)]) - - # index re-creation - for type in ("crypter", "container", "hoster", "captcha", "accounts"): - self.plugins[type] = self.parse(type) - - if "accounts" in as_dict: #accounts needs to be reloaded - self.core.accountManager.initPlugins() - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - - return True - - def loadIcons(self): - """Loads all icons from plugins, plugin type is not in result, because its not important here. - - :return: Dict of names mapped to icons - """ - pass - - def loadIcon(self, type, name): - """ load icon for single plugin, base64 encoded""" - pass - - def checkDependencies(self, type, name): - """ Check deps for given plugin - - :return: List of unfullfilled dependencies - """ - pass - diff --git a/module/plugins/hoster/MegauploadCom.py b/module/plugins/hoster/MegauploadCom.py index 336cbfb58..8693e4303 100644 --- a/module/plugins/hoster/MegauploadCom.py +++ b/module/plugins/hoster/MegauploadCom.py @@ -2,17 +2,13 @@ # -*- coding: utf-8 -*- import re -from time import sleep from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL -from module.network.HTTPRequest import BadHeader from module.utils import html_unescape -from module.PyFile import statusMap - -from pycurl import error +from datatypes.PyFile import statusMap def getInfo(urls): yield [(url, 0, 1, url) for url in urls] -- cgit v1.2.3 From 122d9e78da3b41858411b097100246fde628e0a8 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Thu, 9 Aug 2012 21:31:53 +0200 Subject: refined draft --- module/plugins/Base.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/plugins') diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 905744c9e..c97c42328 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -50,6 +50,7 @@ class Base(object): #: Config definition: list of (name, type, verbose_name, default_value) or #: (name, type, verbose_name, short_description, default_value) __config__ = list() + __label__ = "" #TODO: default should be name, makes long_desc obsolete? #: Short description, one liner __description__ = "" #: More detailed text -- cgit v1.2.3 From 54bc92b4c5e0b3543a313f497cbc2276403c5980 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 10 Sep 2012 11:49:35 +0200 Subject: changed config + progress api --- module/plugins/Base.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Base.py b/module/plugins/Base.py index c97c42328..9f6499985 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -47,14 +47,13 @@ class Base(object): __pattern__ = r"" #: Internal addon plugin which is always loaded __internal__ = False - #: Config definition: list of (name, type, verbose_name, default_value) or - #: (name, type, verbose_name, short_description, default_value) + #: Config definition: list of (name, type, label, default_value) or + #: (name, type, label, short_description, default_value) __config__ = list() - __label__ = "" #TODO: default should be name, makes long_desc obsolete? #: Short description, one liner - __description__ = "" + __label__ = "" #: More detailed text - __long_description__ = """""" + __description__ = """""" #: List of needed modules __dependencies__ = tuple() #: Used to assign a category to addon plugins @@ -76,7 +75,7 @@ class Base(object): __author_mail__ = tuple() - def __init__(self, core): + def __init__(self, core, user=None): self.__name__ = self.__class__.__name__ #: Core instance @@ -89,6 +88,8 @@ class Base(object): self.evm = core.eventManager #: :class:`InteractionManager` self.im = core.interactionManager + #: :class:`User`, user related to this plugin + self.user = user #: last interaction task self.task = None -- cgit v1.2.3 From 1a55cb6a2eb8784253410b2e93510b5bcebf7f41 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 10 Sep 2012 15:12:55 +0200 Subject: userApi for plugins --- module/plugins/Base.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) (limited to 'module/plugins') diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 9f6499985..2b9e12653 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -88,8 +88,18 @@ class Base(object): self.evm = core.eventManager #: :class:`InteractionManager` self.im = core.interactionManager - #: :class:`User`, user related to this plugin - self.user = user + if user: + #: :class:`Api`, user api when user is set + self.api = self.core.api.withUserContext(user) + if self.api: + #: :class:`User`, user related to this plugin + self.user = self.api.user + else: + self.api = self.core.api + self.user = None + else: + self.api = self.core.api + self.user = None #: last interaction task self.task = None @@ -176,7 +186,7 @@ class Base(object): def checkAbort(self): """ Will be overwritten to determine if control flow should be aborted """ - if self.abort: raise Abort() + if self.abort(): raise Abort() def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """Load content at url and returns it -- cgit v1.2.3