diff options
Diffstat (limited to 'module/plugins')
50 files changed, 2029 insertions, 1591 deletions
diff --git a/module/plugins/Account.py b/module/plugins/Account.py index c147404e0..780a8ee69 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -1,145 +1,165 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from random import choice from time import time from traceback import print_exc from threading import RLock -from Plugin import Base -from module.utils import compare_time, parseFileSize, lock +from module.utils import compare_time, parseFileSize, lock, from_string +from module.Api import AccountInfo +from module.network.CookieJar import CookieJar + +from Base import Base class WrongPassword(Exception): pass - -class Account(Base): +#noinspection PyUnresolvedReferences +class Account(Base, AccountInfo): """ Base class for every Account plugin. Just overwrite `login` and cookies will be stored and account becomes accessible in\ - associated hoster plugin. Plugin should also provide `loadAccountInfo` + associated hoster plugin. Plugin should also provide `loadAccountInfo`. \ + A instance of this class is created for every entered account, it holds all \ + fields of AccountInfo ttype, and can be set easily at runtime. """ - __name__ = "Account" - __version__ = "0.2" - __type__ = "account" - __description__ = """Account Plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + + # Default values + valid = True + validuntil = None + trafficleft = None + maxtraffic = None + premium = True + activated = True #: after that time [in minutes] pyload will relogin the account login_timeout = 600 #: account data will be reloaded after this time info_threshold = 600 + # known options + known_opt = ("time", "limitDL") - def __init__(self, manager, accounts): + + def __init__(self, manager, loginname, password, options): Base.__init__(self, manager.core) + if "activated" in options: + activated = from_string(options["activated"], "bool") + else: + activated = Account.activated + + for opt in self.known_opt: + if opt not in options: + options[opt] = "" + + for opt in options.keys(): + if opt not in self.known_opt: + del options[opt] + + # default account attributes + AccountInfo.__init__(self, self.__name__, loginname, Account.valid, Account.validuntil, Account.trafficleft, + Account.maxtraffic, Account.premium, activated, options) + self.manager = manager - self.accounts = {} - self.infos = {} # cache for account information + self.lock = RLock() + self.timestamp = 0 + self.login_ts = 0 # timestamp for login + self.cj = CookieJar(self.__name__) + self.password = password + self.error = None - self.timestamps = {} - self.setAccounts(accounts) self.init() def init(self): pass - def login(self, user, data, req): + def login(self, req): """login into account, the cookies will be saved so user can be recognized - :param user: loginname - :param data: data dictionary :param req: `Request` instance """ - pass + raise NotImplemented + + def relogin(self): + """ Force a login. """ + req = self.getAccountRequest() + try: + return self._login(req) + finally: + req.close() @lock - def _login(self, user, data): + def _login(self, req): # set timestamp for login - self.timestamps[user] = time() - - req = self.getAccountRequest(user) + self.login_ts = time() + try: - self.login(user, data, req) + try: + self.login(req) + except TypeError: #TODO: temporary + self.logDebug("Deprecated .login(...) signature ommit user, data") + self.login(self.loginname, {"password": self.password}, req) + + + self.valid = True except WrongPassword: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": _("Wrong Password")}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname + , "msg": _("Wrong Password")}) + self.valid = False except Exception, e: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": e}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname + , "msg": e}) + self.valid = False if self.core.debug: print_exc() - finally: - if req: req.close() - - def relogin(self, user): - req = self.getAccountRequest(user) - if req: - req.cj.clear() - req.close() - if user in self.infos: - del self.infos[user] #delete old information - self._login(user, self.accounts[user]) + return self.valid - def setAccounts(self, accounts): - self.accounts = accounts - for user, data in self.accounts.iteritems(): - self._login(user, data) - self.infos[user] = {} + def restoreDefaults(self): + self.validuntil = Account.validuntil + self.trafficleft = Account.trafficleft + self.maxtraffic = Account.maxtraffic + self.premium = Account.premium - def updateAccounts(self, user, password=None, options={}): + def update(self, password=None, options=None): """ updates account and return true if anything changed """ - if user in self.accounts: - self.accounts[user]["valid"] = True #do not remove or accounts will not login - if password: - self.accounts[user]["password"] = password - self.relogin(user) - return True - if options: - before = self.accounts[user]["options"] - self.accounts[user]["options"].update(options) - return self.accounts[user]["options"] != before - else: - self.accounts[user] = {"password": password, "options": options, "valid": True} - self._login(user, self.accounts[user]) + self.login_ts = 0 + self.valid = True #set valid so it will be retried to login + + if "activated" in options: + self.activated = from_string(options["avtivated"], "bool") + + if password: + self.password = password + self.relogin() return True + if options: + # remove unknown options + for opt in options.keys(): + if opt not in self.known_opt: + del options[opt] - def removeAccount(self, user): - if user in self.accounts: - del self.accounts[user] - if user in self.infos: - del self.infos[user] - if user in self.timestamps: - del self.timestamps[user] + before = self.options + self.options.update(options) + return self.options != before + + def getAccountRequest(self): + return self.core.requestFactory.getRequest(self.__name__, self.cj) + + def getDownloadSettings(self): + """ Can be overwritten to change download settings. Default is no chunkLimit, multiDL, resumeDownload + + :return: (chunkLimit, multiDL, resumeDownload) / (int,bool,bool) + """ + return -1, True, True @lock - def getAccountInfo(self, name, force=False): + def getAccountInfo(self, force=False): """retrieve account infos for an user, do **not** overwrite this method!\\ just use it to retrieve infos in hoster plugins. see `loadAccountInfo` @@ -147,113 +167,76 @@ class Account(Base): :param force: reloads cached account information :return: dictionary with information """ - data = Account.loadAccountInfo(self, name) - - if force or name not in self.infos: - self.logDebug("Get Account Info for %s" % name) - req = self.getAccountRequest(name) + if force or self.timestamp + self.info_threshold * 60 < time(): + # make sure to login + req = self.getAccountRequest() + self.checkLogin(req) + self.logDebug("Get Account Info for %s" % self.loginname) try: - infos = self.loadAccountInfo(name, req) - if not type(infos) == dict: - raise Exception("Wrong return format") + try: + infos = self.loadAccountInfo(req) + except TypeError: #TODO: temporary + self.logDebug("Deprecated .loadAccountInfo(...) signature, ommit user argument.") + infos = self.loadAccountInfo(self.loginname, req) except Exception, e: infos = {"error": str(e)} - - if req: req.close() + finally: + req.close() self.logDebug("Account Info: %s" % str(infos)) - - infos["timestamp"] = time() - self.infos[name] = infos - elif "timestamp" in self.infos[name] and self.infos[name][ - "timestamp"] + self.info_threshold * 60 < time(): - self.logDebug("Reached timeout for account data") - self.scheduleRefresh(name) - - data.update(self.infos[name]) - return data - - def isPremium(self, user): - info = self.getAccountInfo(user) - return info["premium"] - - def loadAccountInfo(self, name, req=None): - """this should be overwritten in account plugin,\ - and retrieving account information for user - - :param name: - :param req: `Request` instance + self.timestamp = time() + + self.restoreDefaults() # reset to initial state + if type(infos) == dict: # copy result from dict to class + for k, v in infos.iteritems(): + if hasattr(self, k): + setattr(self, k, v) + else: + self.logDebug("Unknown attribute %s=%s" % (k, v)) + + #TODO: remove user + def loadAccountInfo(self, req): + """ Overwrite this method and set account attributes within this method. + + :param user: Deprecated + :param req: Request instance :return: """ - return { - "validuntil": None, # -1 for unlimited - "login": name, - #"password": self.accounts[name]["password"], #@XXX: security - "options": self.accounts[name]["options"], - "valid": self.accounts[name]["valid"], - "trafficleft": None, # in kb, -1 for unlimited - "maxtraffic": None, - "premium": True, #useful for free accounts - "timestamp": 0, #time this info was retrieved - "type": self.__name__, - } - - def getAllAccounts(self, force=False): - return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()] - - def getAccountRequest(self, user=None): - if not user: - user, data = self.selectAccount() - if not user: - return None - - req = self.core.requestFactory.getRequest(self.__name__, user) - return req - - def getAccountCookies(self, user=None): - if not user: - user, data = self.selectAccount() - if not user: - return None - - cj = self.core.requestFactory.getCookieJar(self.__name__, user) - return cj - - def getAccountData(self, user): - return self.accounts[user] + pass - def selectAccount(self): - """ returns an valid account name and data""" - usable = [] - for user, data in self.accounts.iteritems(): - if not data["valid"]: continue + def getAccountCookies(self, user): + self.logDebug("Deprecated method .getAccountCookies -> use account.cj") + return self.cj - if "time" in data["options"] and data["options"]["time"]: - time_data = "" - try: - time_data = data["options"]["time"][0] - start, end = time_data.split("-") - if not compare_time(start.split(":"), end.split(":")): - continue - except: - self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + def getAccountData(self, user): + self.logDebug("Deprecated method .getAccountData -> use fields directly") + return {"password": self.password} - if user in self.infos: - if "validuntil" in self.infos[user]: - if self.infos[user]["validuntil"] > 0 and time() > self.infos[user]["validuntil"]: - continue - if "trafficleft" in self.infos[user]: - if self.infos[user]["trafficleft"] == 0: - continue + def isPremium(self, user=None): + if user: self.logDebug("Deprecated Argument user for .isPremium()", user) + return self.premium - usable.append((user, data)) + def isUsable(self): + """Check several contraints to determine if account should be used""" + if not self.valid or not self.activated: return False - if not usable: return None, None - return choice(usable) + if self.options["time"]: + time_data = "" + try: + time_data = self.options["time"] + start, end = time_data.split("-") + if not compare_time(start.split(":"), end.split(":")): + return False + except: + self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + + if 0 < self.validuntil < time(): + return False + if self.trafficleft is 0: # test explicity for 0 + return False - def canUse(self): - return False if self.selectAccount() == (None, None) else True + return True def parseTraffic(self, string): #returns kbyte return parseFileSize(string) / 1024 @@ -261,32 +244,36 @@ class Account(Base): def wrongPassword(self): raise WrongPassword - def empty(self, user): - if user in self.infos: - self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user) + def empty(self, user=None): + if user: self.logDebug("Deprecated argument user for .empty()", user) + + self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % self.login) - self.infos[user].update({"trafficleft": 0}) - self.scheduleRefresh(user, 30 * 60) + self.trafficleft = 0 + self.scheduleRefresh(30 * 60) def expired(self, user): if user in self.infos: self.logWarning(_("Account %s is expired, checking again in 1h") % user) - self.infos[user].update({"validuntil": time() - 1}) - self.scheduleRefresh(user, 60 * 60) + self.validuntil = time() - 1 + self.scheduleRefresh(60 * 60) - def scheduleRefresh(self, user, time=0, force=True): + def scheduleRefresh(self, time=0, force=True): """ add task to refresh account info to sheduler """ - self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) - self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) + self.logDebug("Scheduled Account refresh for %s in %s seconds." % (self.loginname, time)) + self.core.scheduler.addJob(time, self.getAccountInfo, [force]) @lock - def checkLogin(self, user): + def checkLogin(self, req): """ checks if user is still logged in """ - if user in self.timestamps: - if self.timestamps[user] + self.login_timeout * 60 < time(): - self.logDebug("Reached login timeout for %s" % user) - self.relogin(user) - return False + if self.login_ts + self.login_timeout * 60 < time(): + if self.login_ts: # seperate from fresh login to have better debug logs + self.logDebug("Reached login timeout for %s" % self.loginname) + else: + self.logDebug("Login with %s" % self.loginname) + + self._login(req) + return False return True diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index fc521d36c..c610d10e0 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -17,169 +17,125 @@ @author: RaNaN """ -from os.path import exists -from shutil import copy - from threading import Lock +from random import choice -from module.PullEvents import AccountUpdateEvent -from module.utils import chmod, lock - -ACC_VERSION = 1 +from module.common.json_layer import json +from module.utils import lock -class AccountManager(): +class AccountManager: """manages all accounts""" - #---------------------------------------------------------------------- def __init__(self, core): """Constructor""" self.core = core self.lock = Lock() - self.initPlugins() - self.saveAccounts() # save to add categories to conf + self.loadAccounts() - def initPlugins(self): - self.accounts = {} # key = ( plugin ) - self.plugins = {} + def loadAccounts(self): + """loads all accounts available""" - self.initAccountPlugins() - self.loadAccounts() + self.accounts = {} + for plugin, loginname, activated, password, options in self.core.db.loadAccounts(): + # put into options as used in other context + options = json.loads(options) if options else {} + options["activated"] = activated - def getAccountPlugin(self, plugin): - """get account instance for plugin or None if anonymous""" - if plugin in self.accounts: - if plugin not in self.plugins: - self.plugins[plugin] = self.core.pluginManager.loadClass("accounts", plugin)(self, self.accounts[plugin]) + self.createAccount(plugin, loginname, password, options) + + return + + def iterAccounts(self): + """ yields login, account for all accounts""" + for name, data in self.accounts.iteritems(): + for login, account in data.iteritems(): + yield login, account - return self.plugins[plugin] - else: - return None - - def getAccountPlugins(self): - """ get all account instances""" - - plugins = [] - for plugin in self.accounts.keys(): - plugins.append(self.getAccountPlugin(plugin)) - - return plugins - #---------------------------------------------------------------------- - def loadAccounts(self): - """loads all accounts available""" - - if not exists("accounts.conf"): - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - - f = open("accounts.conf", "rb") - content = f.readlines() - version = content[0].split(":")[1].strip() if content else "" - f.close() - - if not version or int(version) < ACC_VERSION: - copy("accounts.conf", "accounts.backup") - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - self.core.log.warning(_("Account settings deleted, due to new config format.")) - return - - - - plugin = "" - name = "" - - for line in content[1:]: - line = line.strip() - - if not line: continue - if line.startswith("#"): continue - if line.startswith("version"): continue - - if line.endswith(":") and line.count(":") == 1: - plugin = line[:-1] - self.accounts[plugin] = {} - - elif line.startswith("@"): - try: - option = line[1:].split() - self.accounts[plugin][name]["options"][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) - except: - pass - - elif ":" in line: - name, sep, pw = line.partition(":") - self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} - #---------------------------------------------------------------------- def saveAccounts(self): """save all account information""" - - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION) + "\n") - - for plugin, accounts in self.accounts.iteritems(): - f.write("\n") - f.write(plugin+":\n") - - for name,data in accounts.iteritems(): - f.write("\n\t%s:%s\n" % (name,data["password"]) ) - if data["options"]: - for option, values in data["options"].iteritems(): - f.write("\t@%s %s\n" % (option, " ".join(values))) - - f.close() - chmod(f.name, 0600) - - - #---------------------------------------------------------------------- - def initAccountPlugins(self): - """init names""" - for name in self.core.pluginManager.getAccountPlugins(): - self.accounts[name] = {} - + + data = [] + for name, plugin in self.accounts.iteritems(): + data.extend([(name, acc.loginname, acc.activated, acc.password, json.dumps(acc.options)) for acc in + plugin.itervalues()]) + self.core.db.saveAccounts(data) + + def createAccount(self, plugin, loginname, password, options): + klass = self.core.pluginManager.loadClass("accounts", plugin) + if not klass: + self.core.log.warning(_("Unknown account plugin %s") % plugin) + return + + if plugin not in self.accounts: + self.accounts[plugin] = {} + + self.core.log.debug("Create account %s:%s" % (plugin, loginname)) + + self.accounts[plugin][loginname] = klass(self, loginname, password, options) + + + def getAccount(self, plugin, user): + return self.accounts[plugin].get(user, None) + @lock - def updateAccount(self, plugin , user, password=None, options={}): + def updateAccount(self, plugin, user, password=None, options={}): """add or update account""" - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - updated = p.updateAccounts(user, password, options) - #since accounts is a ref in plugin self.accounts doesnt need to be updated here - + if plugin in self.accounts and user in self.accounts[plugin]: + acc = self.accounts[plugin][user] + updated = acc.update(password, options) + self.saveAccounts() - if updated: p.scheduleRefresh(user, force=False) - + if updated: acc.scheduleRefresh(force=True) + else: + self.createAccount(plugin, user, password, options) + self.saveAccounts() + + self.sendChange(plugin, user) + @lock def removeAccount(self, plugin, user): """remove account""" - + if plugin in self.accounts and user in self.accounts[plugin]: + del self.accounts[plugin][user] + self.core.db.removeAccount(plugin, user) + self.core.eventManager.dispatchEvent("accountDeleted", plugin, user) + else: + self.core.log.debug("Remove non existing account %s %s" % (plugin, user)) + + + @lock + def getAccountForPlugin(self, plugin): if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - p.removeAccount(user) + accs = [x for x in self.accounts[plugin].values() if x.isUsable()] + if accs: return choice(accs) - self.saveAccounts() + return None @lock - def getAccountInfos(self, force=True, refresh=False): - data = {} + def getAllAccounts(self, refresh=False): + """ Return account info, refresh afterwards if needed + :param refresh: + :return: + """ if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - force = False - - for p in self.accounts.keys(): - if self.accounts[p]: - p = self.getAccountPlugin(p) - data[p.__name__] = p.getAllAccounts(force) - else: - data[p] = [] - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) - return data - - def sendChange(self): - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) + self.core.scheduler.addJob(0, self.core.accountManager.getAllAccounts) + + # load unavailable account info + for p_dict in self.accounts.itervalues(): + for acc in p_dict.itervalues(): + acc.getAccountInfo() + + return self.accounts + + def refreshAllAccounts(self): + """ Force a refresh of every account """ + for p in self.accounts.itervalues(): + for acc in p.itervalues(): + acc.getAccountInfo(True) + + def sendChange(self, plugin, name): + self.core.eventManager.dispatchEvent("accountUpdated", plugin, name)
\ No newline at end of file diff --git a/module/plugins/Base.py b/module/plugins/Base.py new file mode 100644 index 000000000..34074095e --- /dev/null +++ b/module/plugins/Base.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" + +import sys +from module.utils import decode +from module.utils.fs import exists, makedirs, join + +# TODO +# more attributes if needed +# get rid of catpcha & container plugins ?! (move to crypter & internals) +# adapt old plugins as needed + +class Fail(Exception): + """ raised when failed """ + +class Retry(Exception): + """ raised when start again from beginning """ + +class Base(object): + """ + The Base plugin class with all shared methods and every possible attribute for plugin definition. + """ + __version__ = "0.1" + #: Regexp pattern which will be matched for download/crypter plugins + __pattern__ = r"" + #: Config definition: list of (name, type, verbose_name, default_value) or + #: (name, type, verbose_name, short_description, default_value) + __config__ = list() + #: Short description, one liner + __description__ = "" + #: More detailed text + __long_description__ = """""" + #: List of needed modules + __dependencies__ = tuple() + #: Tags to categorize the plugin + __tags__ = tuple() + #: Base64 encoded .png icon, please don't use sizes above ~3KB + __icon__ = "" + #: Alternative, link to png icon + __icon_url__ = "" + #: Url with general information/support/discussion + __url__ = "" + #: Url to term of content, user is accepting these when using the plugin + __toc_url__ = "" + #: Url to service (to buy premium) for accounts + __ref_url__ = "" + + __author_name__ = tuple() + __author_mail__ = tuple() + + + def __init__(self, core): + self.__name__ = self.__class__.__name__ + + #: Core instance + self.core = core + #: logging instance + self.log = core.log + #: core config + self.config = core.config + #: :class:`EventManager` + self.evm = core.eventManager + #: :class:`InteractionManager` + self.im = core.interActionManager + + def logInfo(self, *args, **kwargs): + """ Print args to log at specific level + + :param args: Arbitary object which should be logged + :param kwargs: sep=(how to seperate arguments), default = " | " + """ + self._log("info", *args, **kwargs) + + def logWarning(self, *args, **kwargs): + self._log("warning", *args, **kwargs) + + def logError(self, *args, **kwargs): + self._log("error", *args, **kwargs) + + def logDebug(self, *args, **kwargs): + self._log("debug", *args, **kwargs) + + def _log(self, level, *args, **kwargs): + if "sep" in kwargs: + sep = "%s" % kwargs["sep"] + else: + sep = " | " + + + strings = [] + for obj in args: + if type(obj) == unicode: + strings.append(obj) + elif type(obj) == str: + strings.append(decode(obj)) + else: + strings.append(str(obj)) + + getattr(self.log, level)("%s: %s" % (self.__name__, sep.join(strings))) + + def setConfig(self, option, value): + """ Set config value for current plugin + + :param option: + :param value: + """ + self.core.config.set(self.__name__, option, value) + + def getConf(self, option): + """ see `getConfig` """ + return self.core.config.get(self.__name__, option) + + def getConfig(self, option): + """ Returns config value for current plugin + + :param option: + :return: + """ + return self.getConf(option) + + def setStorage(self, key, value): + """ Saves a value persistently to the database """ + self.core.db.setStorage(self.__name__, key, value) + + def store(self, key, value): + """ same as `setStorage` """ + self.core.db.setStorage(self.__name__, key, value) + + def getStorage(self, key=None, default=None): + """ Retrieves saved value or dict of all saved entries if key is None """ + if key is not None: + return self.core.db.getStorage(self.__name__, key) or default + return self.core.db.getStorage(self.__name__, key) + + def retrieve(self, *args, **kwargs): + """ same as `getStorage` """ + return self.getStorage(*args, **kwargs) + + def delStorage(self, key): + """ Delete entry in db """ + self.core.db.delStorage(self.__name__, key) + + def shell(self): + """ open ipython shell """ + if self.core.debug: + from IPython import embed + #noinspection PyUnresolvedReferences + sys.stdout = sys._stdout + embed() + + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): + """Load content at url and returns it + + :param url: url as string + :param get: GET as dict + :param post: POST as dict, list or string + :param ref: Set HTTP_REFERER header + :param cookies: use saved cookies + :param just_header: if True only the header will be retrieved and returned as dict + :param decode: Wether to decode the output according to http header, should be True in most cases + :return: Loaded content + """ + if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") + + res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) + + if self.core.debug: + from inspect import currentframe + + frame = currentframe() + if not exists(join("tmp", self.__name__)): + makedirs(join("tmp", self.__name__)) + + f = open( + join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + , "wb") + del frame # delete the frame or it wont be cleaned + + try: + tmp = res.encode("utf8") + except: + tmp = res + + f.write(tmp) + f.close() + + if just_header: + #parse header + header = {"code": self.req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: continue + + key, none, value = line.partition(":") + key = key.lower().strip() + value = value.strip() + + if key in header: + if type(header[key]) == list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value + res = header + + return res + + def fail(self, reason): + """ fail and give reason """ + raise Fail(reason)
\ No newline at end of file diff --git a/module/plugins/Container.py b/module/plugins/Container.py deleted file mode 100644 index c233d3710..000000000 --- a/module/plugins/Container.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Crypter import Crypter - -from os.path import join, exists, basename -from os import remove -import re - -class Container(Crypter): - __name__ = "Container" - __version__ = "0.1" - __pattern__ = None - __type__ = "container" - __description__ = """Base container plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - - def preprocessing(self, thread): - """prepare""" - - self.setup() - self.thread = thread - - self.loadToDisk() - - self.decrypt(self.pyfile) - self.deleteTmp() - - self.createPackages() - - - def loadToDisk(self): - """loads container to disk if its stored remotely and overwrite url, - or check existent on several places at disk""" - - if self.pyfile.url.startswith("http"): - self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] - content = self.load(self.pyfile.url) - self.pyfile.url = join(self.config["general"]["download_folder"], self.pyfile.name) - f = open(self.pyfile.url, "wb" ) - f.write(content) - f.close() - - else: - self.pyfile.name = basename(self.pyfile.url) - if not exists(self.pyfile.url): - if exists(join(pypath, self.pyfile.url)): - self.pyfile.url = join(pypath, self.pyfile.url) - else: - self.fail(_("File not exists.")) - - - def deleteTmp(self): - if self.pyfile.name.startswith("tmp_"): - remove(self.pyfile.url) - - diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index d1549fe80..6079ae8f6 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -1,72 +1,259 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Plugin import Plugin - -class Crypter(Plugin): - __name__ = "Crypter" - __version__ = "0.1" - __pattern__ = None - __type__ = "container" - __description__ = """Base crypter plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def __init__(self, pyfile): - Plugin.__init__(self, pyfile) - - #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) - self.packages = [] +from traceback import print_exc + +from module.Api import Destination +from module.common.packagetools import parseNames +from module.utils import to_list, has_method, uniqify +from module.utils.fs import exists, remove, fs_encode + +from Base import Base, Retry + +class Package: + """ Container that indicates new package should be created """ + def __init__(self, name, urls=None, dest=Destination.Queue): + self.name = name + self.urls = urls if urls else [] + self.dest = dest + + def addUrl(self, url): + self.urls.append(url) + + def __eq__(self, other): + return self.name == other.name and self.urls == other.urls + + def __repr__(self): + return u"<CrypterPackage name=%s, links=%s, dest=%s" % (self.name, self.urls, self.dest) + + def __hash__(self): + return hash(self.name) ^ hash(frozenset(self.urls)) ^ hash(self.dest) + +class PyFileMockup: + """ Legacy class needed by old crypter plugins """ + def __init__(self, url, pack): + self.url = url + self.name = url + self._package = pack + self.packageid = pack.id if pack else -1 + + def package(self): + return self._package + +class Crypter(Base): + """ + Base class for (de)crypter plugins. Overwrite decrypt* methods. + + How to use decrypt* methods: + + You have to overwrite at least one method of decryptURL, decryptURLs, decryptFile. - #: List of urls, pyLoad will generate packagenames + After decrypting and generating urls/packages you have to return the result. + Valid return Data is: + + :class:`Package` instance Crypter.Package + A **new** package will be created with the name and the urls of the object. + + List of urls and `Package` instances + All urls in the list will be added to the **current** package. For each `Package`\ + instance a new package will be created. + + """ + + #: Prefix to annotate that the submited string for decrypting is indeed file content + CONTENT_PREFIX = "filecontent:" + + @classmethod + def decrypt(cls, core, url_or_urls): + """Static method to decrypt, something. Can be used by other plugins. + To decrypt file content prefix the string with ``CONTENT_PREFIX `` as seen above. + + :param core: pyLoad `Core`, needed in decrypt context + :param url_or_urls: List of urls or single url/ file content + :return: List of decrypted urls, all package info removed + """ + urls = to_list(url_or_urls) + p = cls(core) + try: + result = p.processDecrypt(urls) + finally: + p.clean() + + ret = [] + + for url_or_pack in result: + if isinstance(url_or_pack, Package): #package + ret.extend(url_or_pack.urls) + else: # single url + ret.append(url_or_pack) + # eliminate duplicates + return uniqify(ret) + + def __init__(self, core, package=None, password=None): + Base.__init__(self, core) + self.req = core.requestFactory.getRequest(self.__name__) + + # Package the plugin was initialized for, dont use this, its not guaranteed to be set + self.package = package + #: Password supplied by user + self.password = password + #: Propose a renaming of the owner package + self.rename = None + + # For old style decrypter, do not use these ! + self.packages = [] self.urls = [] - - self.multiDL = True - self.limitDL = 0 - - - def preprocessing(self, thread): - """prepare""" - self.setup() - self.thread = thread - - self.decrypt(self.pyfile) - - self.createPackages() - - - def decrypt(self, pyfile): + self.pyfile = None + + self.init() + + def init(self): + """More init stuff if needed""" + + def setup(self): + """Called everytime before decrypting. A Crypter plugin will be most likly used for several jobs.""" + + def decryptURL(self, url): + """Decrypt a single url + + :param url: url to decrypt + :return: See :class:`Crypter` Documentation + """ + if url.startswith("http"): # basic method to redirect + return self.decryptFile(self.load(url)) + else: + self.fail(_("Not existing file or unsupported protocol")) + + def decryptURLs(self, urls): + """Decrypt a bunch of urls + + :param urls: list of urls + :return: See :class:`Crypter` Documentation + """ raise NotImplementedError - def createPackages(self): - """ create new packages from self.packages """ - for pack in self.packages: + def decryptFile(self, content): + """Decrypt file content - self.log.debug("Parsed package %(name)s with %(len)d links" % { "name" : pack[0], "len" : len(pack[1]) } ) - - links = [x.decode("utf-8") for x in pack[1]] - - pid = self.core.api.addPackage(pack[0], links, self.pyfile.package().queue) + :param content: content to decrypt as string + :return: See :class:`Crypter` Documentation + """ + raise NotImplementedError + + def generatePackages(self, urls): + """Generates :class:`Package` instances and names from urls. Usefull for many different links and no\ + given package name. + + :param urls: list of urls + :return: list of `Package` + """ + return [Package(name, purls) for name, purls in parseNames([(url,url) for url in urls]).iteritems()] + + def _decrypt(self, urls): + """ Internal method to select decrypting method + + :param urls: List of urls/content + :return: + """ + cls = self.__class__ + + # seperate local and remote files + content, urls = self.getLocalContent(urls) + + if has_method(cls, "decryptURLs"): + self.setup() + result = to_list(self.decryptURLs(urls)) + elif has_method(cls, "decryptURL"): + result = [] + for url in urls: + self.setup() + result.extend(to_list(self.decryptURL(url))) + elif has_method(cls, "decrypt"): + self.logDebug("Deprecated .decrypt() method in Crypter plugin") + result = [] + for url in urls: + self.pyfile = PyFileMockup(url, self.package) + self.setup() + self.decrypt(self.pyfile) + result.extend(self.convertPackages()) + else: + if not has_method(cls, "decryptFile") or urls: + self.logDebug("No suited decrypting method was overwritten in plugin") + result = [] + + if has_method(cls, "decryptFile"): + for f, c in content: + self.setup() + result.extend(to_list(self.decryptFile(c))) + try: + if f.startswith("tmp_"): remove(f) + except : + pass + + return result + + def processDecrypt(self, urls): + """Catches all exceptions in decrypt methods and return results + + :return: Decrypting results + """ + try: + return self._decrypt(urls) + except Exception: + if self.core.debug: + print_exc() + return [] + + def getLocalContent(self, urls): + """Load files from disk and seperate to file content and url list + + :param urls: + :return: list of (filename, content), remote urls + """ + content = [] + # do nothing if no decryptFile method + if hasattr(self.__class__, "decryptFile"): + remote = [] + for url in urls: + path = None + if url.startswith("http"): # skip urls directly + pass + elif url.startswith(self.CONTENT_PREFIX): + path = url + elif exists(url): + path = url + elif exists(self.core.path(url)): + path = self.core.path(url) + + if path: + try: + if path.startswith(self.CONTENT_PREFIX): + content.append(("", path[len(self.CONTENT_PREFIX)])) + else: + f = open(fs_encode(path), "rb") + content.append((f.name, f.read())) + f.close() + except IOError, e: + self.logError("IOError", e) + else: + remote.append(url) + + #swap filtered url list + urls = remote + + return content, urls - if self.pyfile.package().password: - self.core.api.setPackageData(pid, {"password": self.pyfile.package().password}) + def retry(self): + """ Retry decrypting, will only work once. Somewhat deprecated method, should be avoided. """ + raise Retry() - if self.urls: - self.core.api.generateAndAddPackages(self.urls) + def convertPackages(self): + """ Deprecated """ + self.logDebug("Deprecated method .convertPackages()") + res = [Package(name, urls) for name, urls in self.packages] + res.extend(self.urls) + return res + def clean(self): + if hasattr(self, "req"): + self.req.close() + del self.req
\ No newline at end of file diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index 5efd08bae..83ef091ae 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -14,38 +14,50 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - @author: mkaay - @interface-version: 0.2 + @author: RaNaN """ from traceback import print_exc -from Plugin import Base +#from functools import wraps +from module.utils import has_method + +from Base import Base + +def class_name(p): + return p.rpartition(".")[2] class Expose(object): """ used for decoration to declare rpc services """ + def __new__(cls, f, *args, **kwargs): + hookManager.addRPC(class_name(f.__module__), f.func_name, f.func_doc) + return f + +def AddEventListener(event): + """ used to register method for events """ + class _klass(object): + def __new__(cls, f, *args, **kwargs): + hookManager.addEventListener(class_name(f.__module__), f.func_name, event) + return f + return _klass + +class ConfigHandler(object): + """ register method as config handler """ def __new__(cls, f, *args, **kwargs): - hookManager.addRPC(f.__module__, f.func_name, f.func_doc) + hookManager.addConfigHandler(class_name(f.__module__), f.func_name) return f def threaded(f): + #@wraps(f) def run(*args,**kwargs): hookManager.startThread(f, *args, **kwargs) return run class Hook(Base): """ - Base class for hook plugins. + Base class for hook plugins. Please use @threaded decorator for all longer running task. """ - __name__ = "Hook" - __version__ = "0.2" - __type__ = "hook" - __threaded__ = [] - __config__ = [ ("name", "type", "desc" , "default") ] - __description__ = """interface for hook""" - __author_name__ = ("mkaay", "RaNaN") - __author_mail__ = ("mkaay@mkaay.de", "RaNaN@pyload.org") #: automatically register event listeners for functions, attribute will be deleted dont use it yourself event_map = None @@ -75,20 +87,21 @@ class Hook(Base): for event, funcs in self.event_map.iteritems(): if type(funcs) in (list, tuple): for f in funcs: - self.manager.addEvent(event, getattr(self,f)) + self.evm.addEvent(event, getattr(self,f)) else: - self.manager.addEvent(event, getattr(self,funcs)) + self.evm.addEvent(event, getattr(self,funcs)) #delete for various reasons self.event_map = None if self.event_list: for f in self.event_list: - self.manager.addEvent(f, getattr(self,f)) + self.evm.addEvent(f, getattr(self,f)) self.event_list = None self.initPeriodical() + self.init() self.setup() def initPeriodical(self): @@ -108,36 +121,51 @@ class Hook(Base): def __repr__(self): return "<Hook %s>" % self.__name__ - + + def isActivated(self): + """ checks if hook is activated""" + return self.getConfig("activated") + + def init(self): + pass + def setup(self): """ more init stuff if needed """ pass - def unload(self): - """ called when hook was deactivated """ + def activate(self): + """ Used to activate the hook """ + if has_method(self.__class__, "coreReady"): + self.logDebug("Deprecated method .coreReady() use activated() instead") + self.coreReady() + + def deactivate(self): + """ Used to deactivate the hook. """ pass - - def isActivated(self): - """ checks if hook is activated""" - return self.config.getPlugin(self.__name__, "activated") - - #event methods - overwrite these if needed - def coreReady(self): + def periodical(self): pass - def coreExiting(self): + def newCaptchaTask(self, task): + """ new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """ pass - + + def captchaCorrect(self, task): + pass + + def captchaInvalid(self, task): + pass + + # public events starts from here def downloadPreparing(self, pyfile): pass def downloadFinished(self, pyfile): pass - + def downloadFailed(self, pyfile): pass - + def packageFinished(self, pypack): pass @@ -145,17 +173,4 @@ class Hook(Base): pass def afterReconnecting(self, ip): - pass - - def periodical(self): - pass - - def newCaptchaTask(self, task): - """ new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """ - pass - - def captchaCorrect(self, task): - pass - - def captchaInvalid(self, task): pass
\ No newline at end of file diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 814a70949..c30fed412 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -13,21 +13,473 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay + + @author: RaNaN, spoob, mkaay """ -from module.plugins.Plugin import Plugin +from time import time, sleep +from random import randint + +import os + +if os.name != "nt": + from module.utils.fs import chown + from pwd import getpwnam + from grp import getgrnam + +from Base import Base, Fail, Retry +from module.utils import chunks as _chunks +from module.utils.fs import save_join, save_filename, fs_encode, fs_decode,\ + remove, makedirs, chmod, stat, exists, join + +# Import for Hoster Plugins +chunks = _chunks + +class Abort(Exception): + """ raised when aborted """ + +class Reconnect(Exception): + """ raised when reconnected """ + +class SkipDownload(Exception): + """ raised when download should be skipped """ + +class Hoster(Base): + """ + Base plugin for hoster plugin. Overwrite getInfo for online status retrieval, process for downloading. + """ + + @staticmethod + def getInfo(urls): + """This method is used to retrieve the online status of files for hoster plugins. + It has to *yield* list of tuples with the result in this format (name, size, status, url), + where status is one of API pyfile statusses. + + :param urls: List of urls + :return: yield list of tuple with results (name, size, status, url) + """ + pass + + def __init__(self, pyfile): + Base.__init__(self, pyfile.m.core) + + self.wantReconnect = False + #: enables simultaneous processing of multiple downloads + self.limitDL = 0 + #: chunk limit + self.chunkLimit = 1 + #: enables resume (will be ignored if server dont accept chunks) + self.resumeDownload = False + + #: time() + wait in seconds + self.waitUntil = 0 + self.waiting = False + + self.ocr = None #captcha reader instance + #: account handler instance, see :py:class:`Account` + self.account = self.core.accountManager.getAccountForPlugin(self.__name__) + + #: premium status + self.premium = False + #: username/login + self.user = None + + if self.account and not self.account.isUsable(): self.account = None + if self.account: + self.user = self.account.loginname + #: Browser instance, see `network.Browser` + self.req = self.account.getAccountRequest() + # Default: -1, True, True + self.chunkLimit, self.resumeDownload, self.multiDL = self.account.getDownloadSettings() + self.premium = self.account.isPremium() + else: + self.req = self.core.requestFactory.getRequest(self.__name__) + + #: associated pyfile instance, see `PyFile` + self.pyfile = pyfile + self.thread = None # holds thread in future + + #: location where the last call to download was saved + self.lastDownload = "" + #: re match of the last call to `checkDownload` + self.lastCheck = None + #: js engine, see `JsEngine` + self.js = self.core.js + self.cTask = None #captcha task + + self.retries = 0 # amount of retries already made + self.html = None # some plugins store html code here + + self.init() + + def getMultiDL(self): + self.logDebug("Deprectated attribute multiDL, use limitDL instead") + return self.limitDL <= 0 + + def setMultiDL(self, val): + self.logDebug("Deprectated attribute multiDL, use limitDL instead") + self.limitDL = 0 if val else 1 + + multiDL = property(getMultiDL, setMultiDL) + + def getChunkCount(self): + if self.chunkLimit <= 0: + return self.config["download"]["chunks"] + return min(self.config["download"]["chunks"], self.chunkLimit) + + def getDownloadLimit(self): + if self.account: + limit = self.account.options.get("limitDL", 0) + if limit == "": limit = 0 + return int(limit) + else: + return self.limitDL + + + def __call__(self): + return self.__name__ + + def init(self): + """initialize the plugin (in addition to `__init__`)""" + pass + + def setup(self): + """ setup for enviroment and other things, called before downloading (possibly more than one time)""" + pass + + def preprocessing(self, thread): + """ handles important things to do before starting """ + self.thread = thread + + if self.account: + # will force a relogin or reload of account info if necessary + self.account.getAccountInfo() + else: + self.req.clearCookies() + + self.setup() + + self.pyfile.setStatus("starting") + + return self.process(self.pyfile) + + + def process(self, pyfile): + """the 'main' method of every plugin, you **have to** overwrite it""" + raise NotImplementedError + + def resetAccount(self): + """ dont use account and retry download """ + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retry() + + def checksum(self, local_file=None): + """ + return codes: + 0 - checksum ok + 1 - checksum wrong + 5 - can't get checksum + 10 - not implemented + 20 - unknown error + """ + #@TODO checksum check hook + + return True, 10 + + + def setWait(self, seconds, reconnect=False): + """Set a specific wait time later used with `wait` + + :param seconds: wait time in seconds + :param reconnect: True if a reconnect would avoid wait time + """ + if reconnect: + self.wantReconnect = True + self.pyfile.waitUntil = time() + int(seconds) + + def wait(self): + """ waits the time previously set """ + self.waiting = True + self.pyfile.setStatus("waiting") + + while self.pyfile.waitUntil > time(): + self.thread.m.reconnecting.wait(2) + + if self.pyfile.abort: raise Abort + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + self.waiting = False + self.pyfile.setStatus("starting") + + def offline(self): + """ fail and indicate file is offline """ + raise Fail("offline") + + def tempOffline(self): + """ fail and indicates file ist temporary offline, the core may take consequences """ + raise Fail("temp. offline") + + def retry(self, max_tries=3, wait_time=1, reason=""): + """Retries and begin again from the beginning + + :param max_tries: number of maximum retries + :param wait_time: time to wait in seconds + :param reason: reason for retrying, will be passed to fail if max_tries reached + """ + if 0 < max_tries <= self.retries: + if not reason: reason = "Max retries reached" + raise Fail(reason) + + self.wantReconnect = False + self.setWait(wait_time) + self.wait() + + self.retries += 1 + raise Retry(reason) + + def invalidCaptcha(self): + if self.cTask: + self.cTask.invalid() + + def correctCaptcha(self): + if self.cTask: + self.cTask.correct() + + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', + result_type='textual'): + """ Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param forceUser: if True, ocr is not used + :param imgtype: Type of the Image + :param result_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + + :return: result of decrypting + """ + + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time())[-6:].replace(".", "") + temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + temp_file.write(img) + temp_file.close() + + has_plugin = self.__name__ in self.core.pluginManager.getPlugins("captcha") + + if self.core.captcha: + Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) + else: + Ocr = None + + if Ocr and not forceUser: + sleep(randint(3000, 5000) / 1000.0) + if self.pyfile.abort: raise Abort + + ocr = Ocr() + result = ocr.get_captcha(temp_file.name) + else: + captchaManager = self.core.captchaManager + task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) + self.cTask = task + captchaManager.handleCaptcha(task) + + while task.isWaiting(): + if self.pyfile.abort: + captchaManager.removeTask(task) + raise Abort + sleep(1) + + captchaManager.removeTask(task) + + if task.error and has_plugin: #ignore default error message since the user could use OCR + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) + + result = task.result + self.log.debug("Received captcha result: %s" % str(result)) + + if not self.core.debug: + try: + remove(temp_file.name) + except: + pass + + return result + + + def load(self, *args, **kwargs): + """ See 'Base' load method for more info """ + if self.pyfile.abort: raise Abort + return Base.load(self, *args, **kwargs) + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + """Downloads the content at url to download folder + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param disposition: if True and server provides content-disposition header\ + the filename will be changed if needed + :return: The location where the file was saved + """ + + self.checkForSameFiles() + + self.pyfile.setStatus("downloading") + + download_folder = self.config['general']['download_folder'] + + location = save_join(download_folder, self.pyfile.package().folder) + + if not exists(location): + makedirs(location, int(self.core.config["permission"]["folder"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(location, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + # convert back to unicode + location = fs_decode(location) + name = save_filename(self.pyfile.name) + + filename = join(location, name) + + self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) + + try: + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, + chunks=self.getChunkCount(), resume=self.resumeDownload, + progressNotify=self.pyfile.setProgress, disposition=disposition) + finally: + self.pyfile.size = self.req.size + + if disposition and newname and newname != name: #triple check, just to be sure + self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = join(location, newname) + + fs_filename = fs_encode(filename) + + if self.core.config["permission"]["change_file"]: + chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(fs_filename, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + self.lastDownload = filename + return self.lastDownload + + def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): + """ checks the content of the last downloaded file, re match is saved to `lastCheck` + + :param rules: dict with names and rules to match (compiled regexp or strings) + :param api_size: expected file size + :param max_size: if the file is larger then it wont be checked + :param delete: delete if matched + :param read_size: amount of bytes to read from files larger then max_size + :return: dictionary key of the first rule that matched + """ + lastDownload = fs_encode(self.lastDownload) + if not exists(lastDownload): return None + + size = stat(lastDownload) + size = size.st_size + + if api_size and api_size <= size: return None + elif size > max_size and not read_size: return None + self.log.debug("Download Check triggered") + f = open(lastDownload, "rb") + content = f.read(read_size if read_size else -1) + f.close() + #produces encoding errors, better log to other file in the future? + #self.log.debug("Content: %s" % content) + for name, rule in rules.iteritems(): + if type(rule) in (str, unicode): + if rule in content: + if delete: + remove(lastDownload) + return name + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + if delete: + remove(lastDownload) + self.lastCheck = m + return name + + + def getPassword(self): + """ get the password the user provided in the package""" + password = self.pyfile.package().password + if not password: return "" + return password + + + def checkForSameFiles(self, starting=False): + """ checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises SkipDownload: + """ + + pack = self.pyfile.package() + + for pyfile in self.core.files.cache.values(): + if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile.status in (0, 12): #finished or downloading + raise SkipDownload(pyfile.pluginname) + elif pyfile.status in ( + 5, 7) and starting: #a download is waiting/starting and was appenrently started before + raise SkipDownload(pyfile.pluginname) + + download_folder = self.config['general']['download_folder'] + location = save_join(download_folder, pack.folder, self.pyfile.name) + + if starting and self.core.config['download']['skip_existing'] and exists(location): + size = os.stat(location).st_size + if size >= self.pyfile.size: + raise SkipDownload("File exists.") + + pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + if pyfile: + if exists(location): + raise SkipDownload(pyfile[0]) -def getInfo(self): - #result = [ .. (name, size, status, url) .. ] - return + self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) -class Hoster(Plugin): - __name__ = "Hoster" - __version__ = "0.1" - __pattern__ = None - __type__ = "hoster" - __description__ = """Base hoster plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + def clean(self): + """ clean everything and remove references """ + if hasattr(self, "pyfile"): + del self.pyfile + if hasattr(self, "req"): + self.req.close() + del self.req + if hasattr(self, "thread"): + del self.thread + if hasattr(self, "html"): + del self.html diff --git a/module/plugins/MultiHoster.py b/module/plugins/MultiHoster.py new file mode 100644 index 000000000..abbc14466 --- /dev/null +++ b/module/plugins/MultiHoster.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +from time import time + +from module.utils import remove_chars + +from Account import Account + +def normalize(domain): + """ Normalize domain/plugin name, so they are comparable """ + return remove_chars(domain.strip().lower(), "-.") + +#noinspection PyUnresolvedReferences +class MultiHoster(Account): + """ + Base class for MultiHoster services. + This is also an Account instance so you should see :class:`Account` and overwrite necessary methods. + Multihoster becomes only active when an Account was entered and the MultiHoster hook was activated. + You need to overwrite `loadHosterList` and a corresponding :class:`Hoster` plugin with the same name should + be available to make your service working. + """ + + #: List of hoster names that will be replaced so pyLoad will recognize them: (orig_name, pyload_name) + replacements = [("freakshare.net", "freakshare.com")] + + #: Load new hoster list every x seconds + hoster_timeout = 300 + + def __init__(self, *args, **kwargs): + + # Hoster list + self.hoster = [] + # Timestamp + self.ts = 0 + + Account.__init__(self, *args, **kwargs) + + def loadHosterList(self, req): + """Load list of supported hoster + + :return: List of domain names + """ + raise NotImplementedError + + + def isHosterUsuable(self, domain): + """ Determine before downloading if hoster should be used. + + :param domain: domain name + :return: True to let the MultiHoster download, False to fallback to default plugin + """ + return True + + def getHosterList(self, force=False): + if self.ts + self.hoster_timeout < time() or force: + req = self.getAccountRequest() + try: + self.hoster = self.loadHosterList(req) + except Exception, e: + self.logError(e) + return [] + finally: + req.close() + + for rep in self.replacements: + if rep[0] in self.hoster: + self.hoster.remove(rep[0]) + if rep[1] not in self.hoster: + self.hoster.append(rep[1]) + + self.ts = time() + + return self.hoster
\ No newline at end of file diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py deleted file mode 100644 index 15bf3971f..000000000 --- a/module/plugins/Plugin.py +++ /dev/null @@ -1,617 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN, spoob, mkaay -""" - -from time import time, sleep -from random import randint - -import os -from os import remove, makedirs, chmod, stat -from os.path import exists, join - -if os.name != "nt": - from os import chown - from pwd import getpwnam - from grp import getgrnam - -from itertools import islice - -from module.utils import save_join, save_path, fs_encode, fs_decode - -def chunks(iterable, size): - it = iter(iterable) - item = list(islice(it, size)) - while item: - yield item - item = list(islice(it, size)) - - -class Abort(Exception): - """ raised when aborted """ - - -class Fail(Exception): - """ raised when failed """ - - -class Reconnect(Exception): - """ raised when reconnected """ - - -class Retry(Exception): - """ raised when start again from beginning """ - - -class SkipDownload(Exception): - """ raised when download should be skipped """ - - -class Base(object): - """ - A Base class with log/config/db methods *all* plugin types can use - """ - - def __init__(self, core): - #: Core instance - self.core = core - #: logging instance - self.log = core.log - #: core config - self.config = core.config - - #log functions - def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - - def setConf(self, option, value): - """ see `setConfig` """ - self.core.config.setPlugin(self.__name__, option, value) - - def setConfig(self, option, value): - """ Set config value for current plugin - - :param option: - :param value: - :return: - """ - self.setConf(option, value) - - def getConf(self, option): - """ see `getConfig` """ - return self.core.config.getPlugin(self.__name__, option) - - def getConfig(self, option): - """ Returns config value for current plugin - - :param option: - :return: - """ - return self.getConf(option) - - def setStorage(self, key, value): - """ Saves a value persistently to the database """ - self.core.db.setStorage(self.__name__, key, value) - - def store(self, key, value): - """ same as `setStorage` """ - self.core.db.setStorage(self.__name__, key, value) - - def getStorage(self, key=None, default=None): - """ Retrieves saved value or dict of all saved entries if key is None """ - if key is not None: - return self.core.db.getStorage(self.__name__, key) or default - return self.core.db.getStorage(self.__name__, key) - - def retrieve(self, *args, **kwargs): - """ same as `getStorage` """ - return self.getStorage(*args, **kwargs) - - def delStorage(self, key): - """ Delete entry in db """ - self.core.db.delStorage(self.__name__, key) - - -class Plugin(Base): - """ - Base plugin for hoster/crypter. - Overwrite `process` / `decrypt` in your subclassed plugin. - """ - __name__ = "Plugin" - __version__ = "0.4" - __pattern__ = None - __type__ = "hoster" - __config__ = [("name", "type", "desc", "default")] - __description__ = """Base Plugin""" - __author_name__ = ("RaNaN", "spoob", "mkaay") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") - - def __init__(self, pyfile): - Base.__init__(self, pyfile.m.core) - - self.wantReconnect = False - #: enables simultaneous processing of multiple downloads - self.multiDL = True - self.limitDL = 0 - #: chunk limit - self.chunkLimit = 1 - self.resumeDownload = False - - #: time() + wait in seconds - self.waitUntil = 0 - self.waiting = False - - self.ocr = None #captcha reader instance - #: account handler instance, see :py:class:`Account` - self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) - - #: premium status - self.premium = False - #: username/login - self.user = None - - if self.account and not self.account.canUse(): self.account = None - if self.account: - self.user, data = self.account.selectAccount() - #: Browser instance, see `network.Browser` - self.req = self.account.getAccountRequest(self.user) - self.chunkLimit = -1 # chunk limit, -1 for unlimited - #: enables resume (will be ignored if server dont accept chunks) - self.resumeDownload = True - self.multiDL = True #every hoster with account should provide multiple downloads - #: premium status - self.premium = self.account.isPremium(self.user) - else: - self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) - - #: associated pyfile instance, see `PyFile` - self.pyfile = pyfile - self.thread = None # holds thread in future - - #: location where the last call to download was saved - self.lastDownload = "" - #: re match of the last call to `checkDownload` - self.lastCheck = None - #: js engine, see `JsEngine` - self.js = self.core.js - self.cTask = None #captcha task - - self.retries = 0 # amount of retries already made - self.html = None # some plugins store html code here - - self.init() - - def getChunkCount(self): - if self.chunkLimit <= 0: - return self.config["download"]["chunks"] - return min(self.config["download"]["chunks"], self.chunkLimit) - - def __call__(self): - return self.__name__ - - def init(self): - """initialize the plugin (in addition to `__init__`)""" - pass - - def setup(self): - """ setup for enviroment and other things, called before downloading (possibly more than one time)""" - pass - - def preprocessing(self, thread): - """ handles important things to do before starting """ - self.thread = thread - - if self.account: - self.account.checkLogin(self.user) - else: - self.req.clearCookies() - - self.setup() - - self.pyfile.setStatus("starting") - - return self.process(self.pyfile) - - - def process(self, pyfile): - """the 'main' method of every plugin, you **have to** overwrite it""" - raise NotImplementedError - - def resetAccount(self): - """ dont use account and retry download """ - self.account = None - self.req = self.core.requestFactory.getRequest(self.__name__) - self.retry() - - def checksum(self, local_file=None): - """ - return codes: - 0 - checksum ok - 1 - checksum wrong - 5 - can't get checksum - 10 - not implemented - 20 - unknown error - """ - #@TODO checksum check hook - - return True, 10 - - - def setWait(self, seconds, reconnect=False): - """Set a specific wait time later used with `wait` - - :param seconds: wait time in seconds - :param reconnect: True if a reconnect would avoid wait time - """ - if reconnect: - self.wantReconnect = True - self.pyfile.waitUntil = time() + int(seconds) - - def wait(self): - """ waits the time previously set """ - self.waiting = True - self.pyfile.setStatus("waiting") - - while self.pyfile.waitUntil > time(): - self.thread.m.reconnecting.wait(2) - - if self.pyfile.abort: raise Abort - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect - - self.waiting = False - self.pyfile.setStatus("starting") - - def fail(self, reason): - """ fail and give reason """ - raise Fail(reason) - - def offline(self): - """ fail and indicate file is offline """ - raise Fail("offline") - - def tempOffline(self): - """ fail and indicates file ist temporary offline, the core may take consequences """ - raise Fail("temp. offline") - - def retry(self, max_tries=3, wait_time=1, reason=""): - """Retries and begin again from the beginning - - :param max_tries: number of maximum retries - :param wait_time: time to wait in seconds - :param reason: reason for retrying, will be passed to fail if max_tries reached - """ - if 0 < max_tries <= self.retries: - if not reason: reason = "Max retries reached" - raise Fail(reason) - - self.wantReconnect = False - self.setWait(wait_time) - self.wait() - - self.retries += 1 - raise Retry(reason) - - def invalidCaptcha(self): - if self.cTask: - self.cTask.invalid() - - def correctCaptcha(self): - if self.cTask: - self.cTask.correct() - - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual'): - """ Loads a captcha and decrypts it with ocr, plugin, user input - - :param url: url of captcha image - :param get: get part for request - :param post: post part for request - :param cookies: True if cookies should be enabled - :param forceUser: if True, ocr is not used - :param imgtype: Type of the Image - :param result_type: 'textual' if text is written on the captcha\ - or 'positional' for captcha where the user have to click\ - on a specific region on the captcha - - :return: result of decrypting - """ - - img = self.load(url, get=get, post=post, cookies=cookies) - - id = ("%.2f" % time())[-6:].replace(".", "") - temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - temp_file.write(img) - temp_file.close() - - has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins - - if self.core.captcha: - Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) - else: - Ocr = None - - if Ocr and not forceUser: - sleep(randint(3000, 5000) / 1000.0) - if self.pyfile.abort: raise Abort - - ocr = Ocr() - result = ocr.get_captcha(temp_file.name) - else: - captchaManager = self.core.captchaManager - task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) - self.cTask = task - captchaManager.handleCaptcha(task) - - while task.isWaiting(): - if self.pyfile.abort: - captchaManager.removeTask(task) - raise Abort - sleep(1) - - captchaManager.removeTask(task) - - if task.error and has_plugin: #ignore default error message since the user could use OCR - self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) - elif task.error: - self.fail(task.error) - elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) - - result = task.result - self.log.debug("Received captcha result: %s" % str(result)) - - if not self.core.debug: - try: - remove(temp_file.name) - except: - pass - - return result - - - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): - """Load content at url and returns it - - :param url: - :param get: - :param post: - :param ref: - :param cookies: - :param just_header: if True only the header will be retrieved and returned as dict - :param decode: Wether to decode the output according to http header, should be True in most cases - :return: Loaded content - """ - if self.pyfile.abort: raise Abort - #utf8 vs decode -> please use decode attribute in all future plugins - if type(url) == unicode: url = str(url) - - res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) - - if self.core.debug: - from inspect import currentframe - - frame = currentframe() - if not exists(join("tmp", self.__name__)): - makedirs(join("tmp", self.__name__)) - - f = open( - join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) - , "wb") - del frame # delete the frame or it wont be cleaned - - try: - tmp = res.encode("utf8") - except: - tmp = res - - f.write(tmp) - f.close() - - if just_header: - #parse header - header = {"code": self.req.code} - for line in res.splitlines(): - line = line.strip() - if not line or ":" not in line: continue - - key, none, value = line.partition(":") - key = key.lower().strip() - value = value.strip() - - if key in header: - if type(header[key]) == list: - header[key].append(value) - else: - header[key] = [header[key], value] - else: - header[key] = value - res = header - - return res - - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): - """Downloads the content at url to download folder - - :param url: - :param get: - :param post: - :param ref: - :param cookies: - :param disposition: if True and server provides content-disposition header\ - the filename will be changed if needed - :return: The location where the file was saved - """ - - self.checkForSameFiles() - - self.pyfile.setStatus("downloading") - - download_folder = self.config['general']['download_folder'] - - location = save_join(download_folder, self.pyfile.package().folder) - - if not exists(location): - makedirs(location, int(self.core.config["permission"]["folder"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - - chown(location, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) - - # convert back to unicode - location = fs_decode(location) - name = save_path(self.pyfile.name) - - filename = join(location, name) - - self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) - - try: - newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, - chunks=self.getChunkCount(), resume=self.resumeDownload, - progressNotify=self.pyfile.setProgress, disposition=disposition) - finally: - self.pyfile.size = self.req.size - - if disposition and newname and newname != name: #triple check, just to be sure - self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) - self.pyfile.name = newname - filename = join(location, newname) - - fs_filename = fs_encode(filename) - - if self.core.config["permission"]["change_file"]: - chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - - chown(fs_filename, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) - - self.lastDownload = filename - return self.lastDownload - - def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): - """ checks the content of the last downloaded file, re match is saved to `lastCheck` - - :param rules: dict with names and rules to match (compiled regexp or strings) - :param api_size: expected file size - :param max_size: if the file is larger then it wont be checked - :param delete: delete if matched - :param read_size: amount of bytes to read from files larger then max_size - :return: dictionary key of the first rule that matched - """ - lastDownload = fs_encode(self.lastDownload) - if not exists(lastDownload): return None - - size = stat(lastDownload) - size = size.st_size - - if api_size and api_size <= size: return None - elif size > max_size and not read_size: return None - self.log.debug("Download Check triggered") - f = open(lastDownload, "rb") - content = f.read(read_size if read_size else -1) - f.close() - #produces encoding errors, better log to other file in the future? - #self.log.debug("Content: %s" % content) - for name, rule in rules.iteritems(): - if type(rule) in (str, unicode): - if rule in content: - if delete: - remove(lastDownload) - return name - elif hasattr(rule, "search"): - m = rule.search(content) - if m: - if delete: - remove(lastDownload) - self.lastCheck = m - return name - - - def getPassword(self): - """ get the password the user provided in the package""" - password = self.pyfile.package().password - if not password: return "" - return password - - - def checkForSameFiles(self, starting=False): - """ checks if same file was/is downloaded within same package - - :param starting: indicates that the current download is going to start - :raises SkipDownload: - """ - - pack = self.pyfile.package() - - for pyfile in self.core.files.cache.values(): - if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: - if pyfile.status in (0, 12): #finished or downloading - raise SkipDownload(pyfile.pluginname) - elif pyfile.status in ( - 5, 7) and starting: #a download is waiting/starting and was appenrently started before - raise SkipDownload(pyfile.pluginname) - - download_folder = self.config['general']['download_folder'] - location = save_join(download_folder, pack.folder, self.pyfile.name) - - if starting and self.core.config['download']['skip_existing'] and exists(location): - size = os.stat(location).st_size - if size >= self.pyfile.size: - raise SkipDownload("File exists.") - - pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) - if pyfile: - if exists(location): - raise SkipDownload(pyfile[0]) - - self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) - - def clean(self): - """ clean everything and remove references """ - if hasattr(self, "pyfile"): - del self.pyfile - if hasattr(self, "req"): - self.req.close() - del self.req - if hasattr(self, "thread"): - del self.thread - if hasattr(self, "html"): - del self.html diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index f3f5f47bc..c345f765f 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -21,24 +21,34 @@ import re import sys from os import listdir, makedirs -from os.path import isfile, join, exists, abspath +from os.path import isfile, join, exists, abspath, basename from sys import version_info -from itertools import chain +from time import time from traceback import print_exc from module.lib.SafeEval import const_eval as literal_eval -from module.ConfigParser import IGNORE +from module.plugins.Base import Base + +from new_collections import namedtuple + +# ignore these plugin configs, mainly because plugins were wiped out +IGNORE = ( + "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('hooks', 'UnRar'), + 'EasyShareCom', 'FlyshareCz' + ) + +PluginTuple = namedtuple("PluginTuple", "version re deps user path") class PluginManager: ROOT = "module.plugins." USERROOT = "userplugins." - TYPES = ("crypter", "container", "hoster", "captcha", "accounts", "hooks", "internal") + TYPES = ("crypter", "hoster", "captcha", "accounts", "hooks", "internal") - PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') - VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)') - CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) - DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') + SINGLE = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?<!")"(?!")|\'|\().*(?:(?<!")"(?!")|\'|\)))', + re.I) + # note the nongreedy character, that means we can not embed list and dicts + MULTI = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*((?:\{|\[|"{3}).*?(?:"""|\}|\]))', re.DOTALL | re.M | re.I) def __init__(self, core): self.core = core @@ -47,15 +57,23 @@ class PluginManager: self.log = core.log self.plugins = {} + self.modules = {} # cached modules + self.history = [] # match history to speedup parsing (type, name) self.createIndex() + + self.core.config.parseValues(self.core.config.PLUGIN) + #register for import hook sys.meta_path.append(self) + def logDebug(self, type, plugin, msg): + self.log.debug("Plugin %s | %s: %s" % (type, plugin, msg)) + def createIndex(self): """create information for all plugins available""" - + # add to path, so we can import from userplugins sys.path.append(abspath("")) if not exists("userplugins"): @@ -64,27 +82,14 @@ class PluginManager: f = open(join("userplugins", "__init__.py"), "wb") f.close() - self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) - - self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") - self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") - self.plugins["hooks"] = self.hookPlugins = self.parse("hooks") - self.plugins["internal"] = self.internalPlugins = self.parse("internal") + a = time() + for type in self.TYPES: + self.plugins[type] = self.parse(type) - self.log.debug("created index of plugins") + self.log.debug("Created index of plugins in %.2f ms", (time() - a) * 1000) - def parse(self, folder, pattern=False, home={}): - """ - returns dict with information - home contains parsed plugins from module. - - { - name : {path, version, config, (pattern, re), (plugin, class)} - } - - """ + def parse(self, folder, home=None): + """ Analyze and parses all plugins in folder """ plugins = {} if home: pfolder = join("userplugins", folder) @@ -100,10 +105,6 @@ class PluginManager: for f in listdir(pfolder): if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - data = open(join(pfolder, f)) - content = data.read() - data.close() - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): continue elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): @@ -111,148 +112,168 @@ class PluginManager: elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): continue + # replace suffix and version tag name = f[:-3] if name[-1] == ".": name = name[:-4] - version = self.VERSION.findall(content) - if version: - version = float(version[0][1]) - else: - version = 0 + plugin = self.parsePlugin(join(pfolder, f), folder, name, home) + if plugin: + plugins[name] = plugin - # home contains plugins from pyload root - if home and name in home: - if home[name]["v"] >= version: - continue + if not home: + temp = self.parse(folder, plugins) + plugins.update(temp) - if name in IGNORE or (folder, name) in IGNORE: - continue + return plugins - plugins[name] = {} - plugins[name]["v"] = version + def parsePlugin(self, filename, folder, name, home=None): + """ Parses a plugin from disk, folder means plugin type in this context. Also sets config. - module = f.replace(".pyc", "").replace(".py", "") + :arg home: dict with plugins, of which the found one will be matched against (according version) + :returns PluginTuple""" - # the plugin is loaded from user directory - plugins[name]["user"] = True if home else False - plugins[name]["name"] = module + data = open(filename, "rb") + content = data.read() + data.close() - if pattern: - pattern = self.PATTERN.findall(content) + attrs = {} + for m in self.SINGLE.findall(content) + self.MULTI.findall(content): + #replace gettext function and eval result + try: + attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) + except: + self.logDebug(folder, name, "Error when parsing: %s" % m[-1]) + return + if not hasattr(Base, "__%s__" % m[0]): + if m[0] != "type": #TODO remove type from all plugins, its not needed + self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) - if pattern: - pattern = pattern[0][1] - else: - pattern = "^unmachtable$" + version = 0 - plugins[name]["pattern"] = pattern + if "version" in attrs: + try: + version = float(attrs["version"]) + except ValueError: + self.logDebug(folder, name, "Invalid version %s" % attrs["version"]) + version = 9 #TODO remove when plugins are fixed, causing update loops + else: + self.logDebug(folder, name, "No version attribute") - try: - plugins[name]["re"] = re.compile(pattern) - except: - self.log.error(_("%s has a invalid pattern.") % name) + # home contains plugins from pyload root + if home and name in home: + if home[name].version >= version: + return + if name in IGNORE or (folder, name) in IGNORE: + return - # internals have no config - if folder == "internal": - self.core.config.deleteConfig(name) - continue + if "pattern" in attrs and attrs["pattern"]: + try: + plugin_re = re.compile(attrs["pattern"]) + except: + self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"]) + plugin_re = None + else: plugin_re = None - config = self.CONFIG.findall(content) - if config: - config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" + deps = attrs.get("dependencies", None) - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] + # create plugin tuple + plugin = PluginTuple(version, plugin_re, deps, bool(home), filename) - if folder == "hooks": - append = True - for item in config: - if item[0] == "activated": append = False - # activated flag missing - if append: config.append(["activated", "bool", "Activated", False]) + # internals have no config + if folder == "internal": + return plugin - try: - self.core.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) + if folder == "hooks" and "config" not in attrs: + attrs["config"] = (["activated", "bool", "Activated", False],) - elif folder == "hooks": #force config creation - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" - config = (["activated", "bool", "Activated", False],) + if "config" in attrs and attrs["config"]: + config = attrs["config"] + desc = attrs.get("description", "") + long_desc = attrs.get("long_description", "") - try: - self.core.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) + if type(config[0]) == tuple: + config = [list(x) for x in config] + else: + config = [list(config)] - if not home: - temp = self.parse(folder, pattern, plugins) - plugins.update(temp) + if folder == "hooks": + append = True + for item in config: + if item[0] == "activated": append = False - return plugins + # activated flag missing + if append: config.insert(0, ("activated", "bool", "Activated", False)) + + try: + self.core.config.addConfigSection(name, name, desc, long_desc, config) + except: + self.logDebug(folder, name, "Invalid config %s" % config) + + return plugin def parseUrls(self, urls): - """parse plugins for given list of urls""" + """parse plugins for given list of urls, seperate to crypter and hoster""" - last = None - res = [] # tupels of (url, plugin) + res = {"hoster": [], "crypter": []} # tupels of (url, plugin) for url in urls: - if type(url) not in (str, unicode, buffer): continue + if type(url) not in (str, unicode, buffer): + self.log.debug("Parsing invalid type %s" % type(url)) + continue + found = False - if last and last[1]["re"].match(url): - res.append((url, last[0])) + for ptype, name in self.history: + if self.plugins[ptype][name].re.match(url): + res[ptype].append((url, name)) + found = (ptype, name) + break + + if found: # found match + if self.history[0] != found: #update history + self.history.remove(found) + self.history.insert(0, found) continue - for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(), - self.containerPlugins.iteritems()): - if value["re"].match(url): - res.append((url, name)) - last = (name, value) - found = True - break + for ptype in ("crypter", "hoster"): + for name, plugin in self.plugins[ptype].iteritems(): + if plugin.re.match(url): + res[ptype].append((url, name)) + self.history.insert(0, (ptype, name)) + del self.history[10:] # cut down to size of 10 + found = True + break if not found: - res.append((url, "BasePlugin")) + res["hoster"].append((url, "BasePlugin")) + + return res["hoster"], res["crypter"] - return res + def getPlugins(self, type): + return self.plugins.get(type, None) - def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): + def findPlugin(self, name, pluginlist=("hoster", "crypter")): for ptype in pluginlist: if name in self.plugins[ptype]: - return self.plugins[ptype][name], ptype + return ptype, self.plugins[ptype][name] return None, None - def getPlugin(self, name, original=False): - """return plugin module from hoster|decrypter|container""" - plugin, type = self.findPlugin(name) - - if not plugin: - self.log.warning("Plugin %s not found." % name) - plugin = self.hosterPlugins["BasePlugin"] - - if "new_module" in plugin and not original: - return plugin["new_module"] - + def getPluginModule(self, name): + """ Decprecated: return plugin module from hoster|crypter""" + self.log.debug("Deprecated method: .getPluginModule()") + type, plugin = self.findPlugin(name) return self.loadModule(type, name) - def getPluginName(self, name): - """ used to obtain new name if other plugin was injected""" - plugin, type = self.findPlugin(name) + def getPluginClass(self, name): + """ return plugin class from hoster|crypter, always the not overwritten one """ + type, plugin = self.findPlugin(name) + return self.loadClass(type, name) - if "new_name" in plugin: - return plugin["new_name"] - - return name + # MultiHoster will overwrite this + getPlugin = getPluginClass def loadModule(self, type, name): """ Returns loaded module for plugin @@ -262,11 +283,12 @@ class PluginManager: """ plugins = self.plugins[type] if name in plugins: - if "module" in plugins[name]: return plugins[name]["module"] + if (type, name) in self.modules: return self.modules[(type, name)] try: - module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]["name"]), globals(), locals(), - plugins[name]["name"]) - plugins[name]["module"] = module #cache import, maybe unneeded + # convert path to python recognizable import + path = basename(plugins[name].path).replace(".pyc", "").replace(".py", "") + module = __import__(self.ROOT + "%s.%s" % (type, path), globals(), locals(), path) + self.modules[(type, name)] = module # cache import, maybe unneeded return module except Exception, e: self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) @@ -278,10 +300,6 @@ class PluginManager: module = self.loadModule(type, name) if module: return getattr(module, name) - def getAccountPlugins(self): - """return list of account plugin names""" - return self.accountPlugins.keys() - def find_module(self, fullname, path=None): #redirecting imports if necesarry if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins @@ -294,10 +312,10 @@ class PluginManager: if type in self.plugins and name in self.plugins[type]: #userplugin is a newer version - if not user and self.plugins[type][name]["user"]: + if not user and self.plugins[type][name].user: return self - #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name]["user"]: + #imported from userdir, but pyloads is newer + if user and not self.plugins[type][name].user: return self @@ -329,7 +347,7 @@ class PluginManager: self.log.debug("Request reload of plugins: %s" % type_plugins) as_dict = {} - for t,n in type_plugins: + for t, n in type_plugins: if t in as_dict: as_dict[t].append(n) else: @@ -342,16 +360,13 @@ class PluginManager: for type in as_dict.iterkeys(): for plugin in as_dict[type]: if plugin in self.plugins[type]: - if "module" in self.plugins[type][plugin]: + if (type, plugin) in self.modules: self.log.debug("Reloading %s" % plugin) - reload(self.plugins[type][plugin]["module"]) + reload(self.modules[(type, plugin)]) - #index creation - self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) - self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") - self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") + # index re-creation + for type in ("crypter", "container", "hoster", "captcha", "accounts"): + self.plugins[type] = self.parse(type) if "accounts" in as_dict: #accounts needs to be reloaded self.core.accountManager.initPlugins() @@ -359,22 +374,21 @@ class PluginManager: return True + def loadIcons(self): + """Loads all icons from plugins, plugin type is not in result, because its not important here. + :return: Dict of names mapped to icons + """ + pass -if __name__ == "__main__": - _ = lambda x: x - pypath = "/home/christian/Projekte/pyload-0.4/module/plugins" - - from time import time - - p = PluginManager(None) - - a = time() - - test = ["http://www.youtube.com/watch?v=%s" % x for x in range(0, 100)] - print p.parseUrls(test) + def loadIcon(self, type, name): + """ load icon for single plugin, base64 encoded""" + pass - b = time() + def checkDependencies(self, type, name): + """ Check deps for given plugin - print b - a, "s" + :return: List of unfullfilled dependencies + """ + pass diff --git a/module/plugins/accounts/FilesonicCom.py b/module/plugins/accounts/FilesonicCom.py index 1207f1b56..1b0104b2a 100644 --- a/module/plugins/accounts/FilesonicCom.py +++ b/module/plugins/accounts/FilesonicCom.py @@ -37,10 +37,10 @@ class FilesonicCom(Account): decode=True) return json_loads(xml)["FSApi_Utility"]["getFilesonicDomainForCurrentIp"]["response"] - def loadAccountInfo(self, user, req): + def loadAccountInfo(self, req): xml = req.load(self.API_URL + "/user?method=getInfo&format=json", - post={"u": user, - "p": self.accounts[user]["password"]}, decode=True) + post={"u": self.loginname, + "p": self.password}, decode=True) self.logDebug("account status retrieved from api %s" % xml) @@ -56,15 +56,16 @@ class FilesonicCom(Account): validuntil = -1 return {"validuntil": validuntil, "trafficleft": -1, "premium": premium} - def login(self, user, data, req): + def login(self, req): domain = self.getDomain(req) post_vars = { - "email": user, - "password": data["password"], + "email": self.loginname, + "password": self.password, "rememberMe": 1 } page = req.load("http://www%s/user/login" % domain, cookies=True, post=post_vars, decode=True) if "Provided password does not match." in page or "You must be logged in to view this page." in page: self.wrongPassword() + diff --git a/module/plugins/accounts/OronCom.py b/module/plugins/accounts/OronCom.py index 793984121..1fe8a4449 100755 --- a/module/plugins/accounts/OronCom.py +++ b/module/plugins/accounts/OronCom.py @@ -23,13 +23,13 @@ from time import strptime, mktime class OronCom(Account): __name__ = "OronCom" - __version__ = "0.12" + __version__ = "0.13" __type__ = "account" __description__ = """oron.com account plugin""" __author_name__ = ("DHMH") __author_mail__ = ("DHMH@pyload.org") - def loadAccountInfo(self, user, req): + def loadAccountInfo(self, req): req.load("http://oron.com/?op=change_lang&lang=german") src = req.load("http://oron.com/?op=my_account").replace("\n", "") validuntil = re.search(r"<td>Premiumaccount läuft bis:</td>\s*<td>(.*?)</td>", src) @@ -47,8 +47,9 @@ class OronCom(Account): tmp = {"validuntil": validuntil, "trafficleft": trafficleft, "premium" : premium} return tmp - def login(self, user, data, req): + def login(self, req): req.load("http://oron.com/?op=change_lang&lang=german") - page = req.load("http://oron.com/login", post={"login": user, "password": data["password"], "op": "login"}) + page = req.load("http://oron.com/login", post={"login": self.loginname, "password": self.password, "op": "login"}) if r'<b class="err">Login oder Passwort falsch</b>' in page: self.wrongPassword() + diff --git a/module/plugins/accounts/Premium4Me.py b/module/plugins/accounts/Premium4Me.py index de4fdc219..6a52cb61a 100644 --- a/module/plugins/accounts/Premium4Me.py +++ b/module/plugins/accounts/Premium4Me.py @@ -1,23 +1,27 @@ -from module.plugins.Account import Account
+# -*- coding: utf-8 -*-
+from module.plugins.MultiHoster import MultiHoster
-class Premium4Me(Account):
+class Premium4Me(MultiHoster):
__name__ = "Premium4Me"
- __version__ = "0.02"
+ __version__ = "0.10"
__type__ = "account"
__description__ = """Premium4.me account plugin"""
__author_name__ = ("RaNaN", "zoidberg")
__author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz")
- def loadAccountInfo(self, user, req):
+ def loadAccountInfo(self, req):
traffic = req.load("http://premium4.me/api/traffic.php?authcode=%s" % self.authcode)
- account_info = {"trafficleft": int(traffic) / 1024,
- "validuntil": -1}
+ account_info = {"trafficleft": int(traffic) / 1024, "validuntil": -1}
return account_info
- def login(self, user, data, req):
- self.authcode = req.load("http://premium4.me/api/getauthcode.php?username=%s&password=%s" % (user, data["password"])).strip()
-
+ def login(self, req):
+ self.authcode = req.load("http://premium4.me/api/getauthcode.php?username=%s&password=%s" % (self.loginname, self.password)).strip()
+
if "wrong username" in self.authcode:
- self.wrongPassword()
\ No newline at end of file + self.wrongPassword()
+
+ def loadHosterList(self, req):
+ page = req.load("http://premium4.me/api/hosters.php?authcode=%s" % self.authcode)
+ return [x.strip() for x in page.replace("\"", "").split(";")]
\ No newline at end of file diff --git a/module/plugins/accounts/RealdebridCom.py b/module/plugins/accounts/RealdebridCom.py index 3137987a9..4a2cf9368 100644 --- a/module/plugins/accounts/RealdebridCom.py +++ b/module/plugins/accounts/RealdebridCom.py @@ -1,15 +1,18 @@ -from module.plugins.Account import Account
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from module.plugins.MultiHoster import MultiHoster
import xml.dom.minidom as dom
-class RealdebridCom(Account):
+class RealdebridCom(MultiHoster):
__name__ = "RealdebridCom"
- __version__ = "0.4"
+ __version__ = "0.5"
__type__ = "account"
__description__ = """Real-Debrid.com account plugin"""
__author_name__ = ("Devirex, Hazzard")
__author_mail__ = ("naibaf_11@yahoo.de")
- def loadAccountInfo(self, user, req):
+ def loadAccountInfo(self, req):
page = req.load("http://real-debrid.com/api/account.php")
xml = dom.parseString(page)
account_info = {"validuntil": int(xml.getElementsByTagName("expiration")[0].childNodes[0].nodeValue),
@@ -17,9 +20,16 @@ class RealdebridCom(Account): return account_info
- def login(self, user, data, req):
- page = req.load("https://real-debrid.com/ajax/login.php?user=%s&pass=%s" % (user, data["password"]))
+ def login(self, req):
+ page = req.load("https://real-debrid.com/ajax/login.php?user=%s&pass=%s" % (self.loginname, self.password))
#page = req.load("https://real-debrid.com/login.html", post={"user": user, "pass": data["password"]}, cookies=True)
if "Your login informations are incorrect" in page:
self.wrongPassword()
+
+
+ def loadHosterList(self, req):
+ https = "https" if self.getConfig("https") else "http"
+ page = req.load(https + "://real-debrid.com/api/hosters.php").replace("\"","").strip()
+
+ return[x.strip() for x in page.split(",") if x.strip()]
\ No newline at end of file diff --git a/module/plugins/accounts/ShareonlineBiz.py b/module/plugins/accounts/ShareonlineBiz.py index 426f5a6a9..4dd398d6d 100644 --- a/module/plugins/accounts/ShareonlineBiz.py +++ b/module/plugins/accounts/ShareonlineBiz.py @@ -23,24 +23,23 @@ import re class ShareonlineBiz(Account): __name__ = "ShareonlineBiz" - __version__ = "0.21" + __version__ = "0.3" __type__ = "account" __description__ = """share-online.biz account plugin""" __author_name__ = ("mkaay") __author_mail__ = ("mkaay@mkaay.de") - def getUserAPI(self, user, req): - src = req.load("http://api.share-online.biz/account.php?username=%s&password=%s&act=userDetails" % (user, self.accounts[user]["password"])) + def getUserAPI(self, req): + src = req.load("http://api.share-online.biz/account.php?username=%s&password=%s&act=userDetails" % (self.loginname, self.password)) info = {} for line in src.splitlines(): - if "=" in line: - key, value = line.split("=") - info[key] = value + key, value = line.split("=") + info[key] = value return info def loadAccountInfo(self, user, req): try: - info = self.getUserAPI(user, req) + info = self.getUserAPI(req) return {"validuntil": int(info["expire_date"]), "trafficleft": -1, "premium": not info["group"] == "Sammler"} except: pass diff --git a/module/plugins/accounts/ZeveraCom.py b/module/plugins/accounts/ZeveraCom.py new file mode 100644 index 000000000..26eac64b6 --- /dev/null +++ b/module/plugins/accounts/ZeveraCom.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*-
+from module.plugins.MultiHoster import MultiHoster
+
+import re
+from time import mktime, strptime
+
+class ZeveraCom(MultiHoster):
+ __name__ = "ZeveraCom"
+ __version__ = "0.11"
+ __type__ = "account"
+ __description__ = """Zevera.com account plugin"""
+ __author_name__ = ("zoidberg")
+ __author_mail__ = ("zoidberg@mujmail.cz")
+
+ api_url = "http://zevera.com/API.ashx"
+
+ def loadAccountInfo(self, req):
+ dataRet = self.loadAPIRequest(req)
+ account_info = {
+ "trafficleft": dataRet['AccountInfo']['AvailableTODAYTrafficForUseInMBytes'] * 1024,
+ "validuntil": -1 #dataRet['AccountInfo']['EndSubscriptionDate']
+ }
+
+ return account_info
+
+ def login(self, req):
+ if self.loadAPIRequest(req, parse = False) == 'Login Error':
+ self.wrongPassword()
+
+ def loadHosterList(self, req):
+ page = req.load("http://www.zevera.com/jDownloader.ashx?cmd=gethosters")
+ return [x.strip() for x in page.replace("\"", "").split(",")]
+
+ def loadAPIRequest(self, req, parse = True, **kwargs):
+ get_dict = {
+ 'cmd': 'download_request',
+ 'login': self.loginname,
+ 'pass': self.password
+ }
+ get_dict.update(kwargs)
+
+ response = req.load(self.api_url, get = get_dict, decode = True)
+ self.logDebug(response)
+ return self.parseAPIRequest(response) if parse else response
+
+ def parseAPIRequest(self, api_response):
+
+ try:
+ arFields = iter(api_response.split('TAG BEGIN DATA#')[1].split('#END DATA')[0].split('#'))
+
+ retData = {
+ 'VersionMajor': arFields.next(),
+ 'VersionMinor': arFields.next(),
+ 'ErrorCode': int(arFields.next()),
+ 'ErrorMessage': arFields.next(),
+ 'Update_Wait': arFields.next()
+ }
+ serverInfo = {
+ 'DateTimeOnServer': mktime(strptime(arFields.next(),"%Y/%m/%d %H:%M:%S")),
+ 'DAY_Traffic_LimitInMBytes': int(arFields.next())
+ }
+ accountInfo = {
+ 'EndSubscriptionDate': mktime(strptime(arFields.next(),"%Y/%m/%d %H:%M:%S")),
+ 'TrafficUsedInMBytesDayToday': int(arFields.next()),
+ 'AvailableEXTRATrafficForUseInMBytes': int(arFields.next()),
+ 'AvailableTODAYTrafficForUseInMBytes': int(arFields.next())
+ }
+ fileInfo = {
+ 'FileID': arFields.next(),
+ 'Title': arFields.next(),
+ 'RealFileName': arFields.next(),
+ 'FileNameOnServer': arFields.next(),
+ 'StorageServerURL': arFields.next(),
+ 'Token': arFields.next(),
+ 'FileSizeInBytes': int(arFields.next()),
+ 'StatusID': int(arFields.next())
+ }
+ progress = {
+ 'BytesReceived': int(arFields.next()),
+ 'TotalBytesToReceive': int(arFields.next()),
+ 'Percentage': arFields.next(),
+ 'StatusText': arFields.next(),
+ 'ProgressText': arFields.next()
+ }
+ fileInfo.update({
+ 'Progress': progress,
+ 'FilePassword': arFields.next(),
+ 'Keywords': arFields.next(),
+ 'ImageURL4Download': arFields.next(),
+ 'CategoryID': arFields.next(),
+ 'CategoryText': arFields.next(),
+ 'Notes': arFields.next()
+ })
+ retData.update({
+ 'ServerInfo': serverInfo,
+ 'AccountInfo': accountInfo,
+ 'FileInfo': fileInfo
+ })
+
+ except Exception, e:
+ self.logError(e)
+ return None
+
+ self.logDebug(retData)
+ return retData
\ No newline at end of file diff --git a/module/plugins/container/CCF.py b/module/plugins/container/CCF.py index 301b033d4..ab7ff1099 100644 --- a/module/plugins/container/CCF.py +++ b/module/plugins/container/CCF.py @@ -4,13 +4,13 @@ import re from urllib2 import build_opener -from module.plugins.Container import Container +from module.plugins.Crypter import Crypter from module.lib.MultipartPostHandler import MultipartPostHandler from os import makedirs from os.path import exists, join -class CCF(Container): +class CCF(Crypter): __name__ = "CCF" __version__ = "0.2" __pattern__ = r"(?!http://).*\.ccf$" diff --git a/module/plugins/container/RSDF.py b/module/plugins/container/RSDF.py index ea5cd67f2..cbc9864b1 100644 --- a/module/plugins/container/RSDF.py +++ b/module/plugins/container/RSDF.py @@ -5,9 +5,9 @@ import base64 import binascii import re -from module.plugins.Container import Container +from module.plugins.Crypter import Crypter -class RSDF(Container): +class RSDF(Crypter): __name__ = "RSDF" __version__ = "0.21" __pattern__ = r".*\.rsdf" diff --git a/module/plugins/crypter/FilesonicComFolder.py b/module/plugins/crypter/FilesonicComFolder.py index b967a74a1..02ae66295 100644 --- a/module/plugins/crypter/FilesonicComFolder.py +++ b/module/plugins/crypter/FilesonicComFolder.py @@ -4,8 +4,6 @@ import re from module.plugins.Crypter import Crypter class FilesonicComFolder(Crypter): - __name__ = "FilesonicComFolder" - __type__ = "crypter" __pattern__ = r"http://(\w*\.)?(sharingmatrix|filesonic|wupload)\.[^/]*/folder/\w+/?" __version__ = "0.11" __description__ = """Filesonic.com/Wupload.com Folder Plugin""" @@ -15,9 +13,8 @@ class FilesonicComFolder(Crypter): FOLDER_PATTERN = r'<table>\s*<caption>Files Folder</caption>(.*?)</table>' LINK_PATTERN = r'<a href="([^"]+)">' - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - + def decryptURL(self, url): + html = self.load(url) new_links = [] folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) @@ -26,6 +23,7 @@ class FilesonicComFolder(Crypter): new_links.extend(re.findall(self.LINK_PATTERN, folder.group(1))) if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) + return new_links else: - self.fail('Could not extract any links')
\ No newline at end of file + self.fail('Could not extract any links') + diff --git a/module/plugins/container/LinkList.py b/module/plugins/crypter/LinkList.py index b9eb4b972..ebfa373eb 100644 --- a/module/plugins/container/LinkList.py +++ b/module/plugins/crypter/LinkList.py @@ -1,25 +1,25 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from module.plugins.Crypter import Crypter, Package -from module.plugins.Container import Container - -class LinkList(Container): +class LinkList(Crypter): __name__ = "LinkList" __version__ = "0.11" __pattern__ = r".+\.txt$" __description__ = """Read Link Lists in txt format""" - __config__ = [("clear", "bool", "Clear Linklist after adding", False)] __author_name__ = ("spoob", "jeix") __author_mail__ = ("spoob@pyload.org", "jeix@hasnomail.com") + # method declaration is needed here + def decryptURL(self, url): + return Crypter.decryptURL(self, url) - def decrypt(self, pyfile): - txt = open(pyfile.url, 'r') - links = txt.readlines() - curPack = "Parsed links from %s" % pyfile.name - - packages = {curPack:[],} + def decryptFile(self, content): + links = content.splitlines() + + curPack = "default" + packages = {curPack:[]} for link in links: link = link.strip() @@ -33,10 +33,8 @@ class LinkList(Container): packages[curPack] = [] continue packages[curPack].append(link) - txt.close() # empty packages fix - delete = [] for key,value in packages.iteritems(): @@ -46,12 +44,12 @@ class LinkList(Container): for key in delete: del packages[key] - if self.getConfig("clear"): - try: - txt = open(pyfile.url, 'wb') - txt.close() - except: - self.log.warning(_("LinkList could not be cleared.")) - + urls = [] + for name, links in packages.iteritems(): - self.packages.append((name, links, name)) + if name == "default": + urls.extend(links) + else: + urls.append(Package(name, links)) + + return urls
\ No newline at end of file diff --git a/module/plugins/crypter/XfilesharingProFolder.py b/module/plugins/crypter/XfilesharingProFolder.py new file mode 100644 index 000000000..8e58c207d --- /dev/null +++ b/module/plugins/crypter/XfilesharingProFolder.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*-
+
+from module.plugins.Crypter import Crypter, Package
+import re
+
+class XfilesharingProFolder(Crypter):
+ __name__ = "XfilesharingProFolder"
+ __type__ = "crypter"
+ __pattern__ = r"http://(?:www\.)?((easybytez|turboupload|uploadville|file4safe|fileband|filebeep|grupload|247upload)\.com|(muchshare|annonhost).net|bzlink.us)/users/.*"
+ __version__ = "0.01"
+ __description__ = """Generic XfilesharingPro Folder Plugin"""
+ __author_name__ = ("zoidberg")
+ __author_mail__ = ("zoidberg@mujmail.cz")
+
+ LINK_PATTERN = r'<div class="link"><a href="([^"]+)" target="_blank">[^<]*</a></div>'
+ SUBFOLDER_PATTERN = r'<TD width="1%"><img src="[^"]*/images/folder2.gif"></TD><TD><a href="([^"]+)"><b>(?!\. \.<)([^<]+)</b></a></TD>'
+
+ def decryptURL(self, url):
+ return self.decryptFile(self.load(url, decode = True))
+
+ def decryptFile(self, html):
+ new_links = []
+
+ new_links.extend(re.findall(self.LINK_PATTERN, html))
+
+ subfolders = re.findall(self.SUBFOLDER_PATTERN, html)
+ #self.logDebug(subfolders)
+ for (url, name) in subfolders:
+ if self.package: name = "%s/%s" % (self.package.name, name)
+ new_links.append(Package(name, [url]))
+
+ if not new_links: self.fail('Could not extract any links')
+
+ return new_links
\ No newline at end of file diff --git a/module/plugins/hooks/ClickAndLoad.py b/module/plugins/hooks/ClickAndLoad.py index 97e5cd57d..fc32d0da8 100644 --- a/module/plugins/hooks/ClickAndLoad.py +++ b/module/plugins/hooks/ClickAndLoad.py @@ -32,7 +32,7 @@ class ClickAndLoad(Hook): __author_name__ = ("RaNaN", "mkaay") __author_mail__ = ("RaNaN@pyload.de", "mkaay@mkaay.de") - def coreReady(self): + def activate(self): self.port = int(self.core.config['webinterface']['port']) if self.core.config['webinterface']['activated']: try: diff --git a/module/plugins/hooks/Ev0InFetcher.py b/module/plugins/hooks/Ev0InFetcher.py index 5941cf38c..0cd3f3226 100644 --- a/module/plugins/hooks/Ev0InFetcher.py +++ b/module/plugins/hooks/Ev0InFetcher.py @@ -40,7 +40,7 @@ class Ev0InFetcher(Hook): results = self.core.pluginManager.parseUrls(links) sortedLinks = {} - for url, hoster in results: + for url, hoster in results[0]: if hoster not in sortedLinks: sortedLinks[hoster] = [] sortedLinks[hoster].append(url) diff --git a/module/plugins/hooks/ExternalScripts.py b/module/plugins/hooks/ExternalScripts.py index 2e77f1dae..39fe2b9f0 100644 --- a/module/plugins/hooks/ExternalScripts.py +++ b/module/plugins/hooks/ExternalScripts.py @@ -14,16 +14,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - @author: mkaay - @interface-version: 0.1 + @author: RaNaN """ import subprocess -from os import listdir, access, X_OK, makedirs -from os.path import join, exists, basename +from os import access, X_OK, makedirs +from os.path import basename from module.plugins.Hook import Hook -from module.utils import save_join +from module.utils.fs import save_join, exists, join, listdir class ExternalScripts(Hook): __name__ = "ExternalScripts" diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index c789495ca..12bd40d1b 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -3,8 +3,7 @@ import sys import os -from os import remove, chmod, makedirs -from os.path import exists, basename, isfile, isdir, join +from os.path import basename, isfile, isdir, join from traceback import print_exc from copy import copy @@ -49,7 +48,7 @@ if os.name != "nt": from pwd import getpwnam from grp import getgrnam -from module.utils import save_join, fs_encode +from module.utils.fs import save_join, fs_encode, exists, remove, chmod, makedirs from module.plugins.Hook import Hook, threaded, Expose from module.plugins.internal.AbstractExtractor import ArchiveError, CRCError, WrongPassword diff --git a/module/plugins/hooks/MultiHoster.py b/module/plugins/hooks/MultiHoster.py new file mode 100644 index 000000000..749f2c104 --- /dev/null +++ b/module/plugins/hooks/MultiHoster.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from types import MethodType + +from module.plugins.MultiHoster import MultiHoster as MultiHosterAccount, normalize +from module.plugins.Hook import Hook, AddEventListener +from module.plugins.PluginManager import PluginTuple + +class MultiHoster(Hook): + __version__ = "0.1" + __description__ = "Gives ability to use MultiHoster services. You need to add your account first." + __config__ = [("activated", "bool", "Activated", True)] + __author_mail__ = ("pyLoad Team",) + __author_mail__ = ("support@pyload.org",) + + #TODO: multiple accounts - multihoster / config options + + def init(self): + + # overwritten plugins + self.plugins = {} + + def addHoster(self, account): + + self.logDebug("New MultiHoster %s" % account.__name__) + + pluginMap = {} + for name in self.core.pluginManager.getPlugins("hoster").keys(): + pluginMap[name.lower()] = name + + supported = [] + new_supported = [] + + for hoster in account.getHosterList(): + name = normalize(hoster) + + if name in pluginMap: + supported.append(pluginMap[name]) + else: + new_supported.append(hoster) + + if not supported and not new_supported: + account.logError(_("No Hoster loaded")) + return + + klass = self.core.pluginManager.getPluginClass(account.__name__) + + # inject plugin plugin + account.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(supported))) + for hoster in supported: + self.plugins[hoster] = klass + + account.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) + + # create new regexp + regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) + + # recreate plugin tuple for new regexp + hoster = self.core.pluginManager.getPlugins("hoster") + p = hoster[account.__name__] + new = PluginTuple(p.version, re.compile(regexp), p.deps, p.user, p.path) + hoster[account.__name__] = new + + + + @AddEventListener("accountDeleted") + def refreshAccounts(self, plugin=None, user=None): + + self.plugins = {} + + for name, account in self.core.accountManager.iterAccounts(): + if isinstance(account, MultiHosterAccount) and account.isUsable(): + self.addHoster(account) + + @AddEventListener("accountUpdated") + def refreshAccount(self, plugin, user): + + account = self.core.accountManager.getAccount(plugin, user) + if isinstance(account, MultiHosterAccount) and account.isUsable(): + self.addHoster(account) + + def activate(self): + self.refreshAccounts() + + # new method for plugin manager + def getPlugin(self2, name): + if name in self.plugins: + return self.plugins[name] + return self2.getPluginClass(name) + + pm = self.core.pluginManager + pm.getPlugin = MethodType(getPlugin, pm, object) + + + def deactivate(self): + #restore state + pm = self.core.pluginManager + pm.getPlugin = pm.getPluginClass + diff --git a/module/plugins/hooks/RealdebridCom.py b/module/plugins/hooks/RealdebridCom.py deleted file mode 100644 index c57e3de52..000000000 --- a/module/plugins/hooks/RealdebridCom.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.network.RequestFactory import getURL -from module.plugins.internal.MultiHoster import MultiHoster - -class RealdebridCom(MultiHoster): - __name__ = "RealdebridCom" - __version__ = "0.4" - __type__ = "hook" - - __config__ = [("activated", "bool", "Activated", "False"), - ("https", "bool", "Enable HTTPS", "False")] - - __description__ = """Real-Debrid.com hook plugin""" - __author_name__ = ("Devirex, Hazzard") - __author_mail__ = ("naibaf_11@yahoo.de") - - replacements = [("freakshare.net", "freakshare.com")] - - def getHoster(self): - https = "https" if self.getConfig("https") else "http" - page = getURL(https + "://real-debrid.com/api/hosters.php").replace("\"","").strip() - - return[x.strip() for x in page.split(",") if x.strip()]
\ No newline at end of file diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py index ce75399c5..230a6e858 100644 --- a/module/plugins/hooks/UpdateManager.py +++ b/module/plugins/hooks/UpdateManager.py @@ -23,7 +23,7 @@ from os import stat from os.path import join, exists from time import time -from module.ConfigParser import IGNORE +from module.plugins.PluginManager import IGNORE from module.network.RequestFactory import getURL from module.plugins.Hook import threaded, Expose, Hook @@ -60,6 +60,11 @@ class UpdateManager(Hook): @threaded def periodical(self): + + if self.core.version.endswith("-dev"): + self.logDebug("No update check performed on dev version.") + return + update = self.checkForUpdate() if update: self.info["pyload"] = True @@ -129,10 +134,10 @@ class UpdateManager(Hook): else: type = prefix - plugins = getattr(self.core.pluginManager, "%sPlugins" % type) + plugins = self.core.pluginManager.getPlugins(type) if name in plugins: - if float(plugins[name]["v"]) >= float(version): + if float(plugins[name].version) >= float(version): continue if name in IGNORE or (type, name) in IGNORE: diff --git a/module/plugins/hooks/XMPPInterface.py b/module/plugins/hooks/XMPPInterface.py index a96adf524..de87433cf 100644 --- a/module/plugins/hooks/XMPPInterface.py +++ b/module/plugins/hooks/XMPPInterface.py @@ -19,7 +19,7 @@ """ from pyxmpp import streamtls -from pyxmpp.all import JID, Message +from pyxmpp.all import JID, Message, Presence from pyxmpp.jabber.client import JabberClient from pyxmpp.interface import implements from pyxmpp.interfaces import * @@ -121,7 +121,26 @@ class XMPPInterface(IRCInterface, JabberClient): in a client session.""" return [ ("normal", self.message), - ] + ] + + def presence_control(self, stanza): + from_jid = unicode(stanza.get_from_jid()) + stanza_type = stanza.get_type() + self.log.debug("pyLoad XMPP: %s stanza from %s" % (stanza_type, + from_jid)) + + if from_jid in self.getConfig("owners"): + return stanza.make_accept_response() + + return stanza.make_deny_response() + + def session_started(self): + self.stream.send(Presence()) + + self.stream.set_presence_handler("subscribe", self.presence_control) + self.stream.set_presence_handler("subscribed", self.presence_control) + self.stream.set_presence_handler("unsubscribe", self.presence_control) + self.stream.set_presence_handler("unsubscribed", self.presence_control) def message(self, stanza): """Message handler for the component.""" @@ -248,4 +267,10 @@ class VersionHandler(object): q.newTextChild(q.ns(), "name", "Echo component") q.newTextChild(q.ns(), "version", "1.0") return iq -
\ No newline at end of file + + def unload(self): + self.log.debug("pyLoad XMPP: unloading") + self.disconnect() + + def deactivate(self): + self.unload() diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 2de47940d..0e9595265 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -1,12 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from urlparse import urlparse -from re import search -from urllib import unquote from module.network.HTTPRequest import BadHeader from module.plugins.Hoster import Hoster -from module.utils import html_unescape, remove_chars +from module.utils import html_unescape class BasePlugin(Hoster): __name__ = "BasePlugin" @@ -62,28 +60,5 @@ class BasePlugin(Hoster): def downloadFile(self, pyfile): - header = self.load(pyfile.url, just_header = True) - #self.logDebug(header) - - if 'location' in header: - self.logDebug("Location: " + header['location']) - url = unquote(header['location']) - else: - url = pyfile.url - - name = html_unescape(urlparse(url).path.split("/")[-1]) - - if 'content-disposition' in header: - self.logDebug("Content-Disposition: " + header['content-disposition']) - m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", header['content-disposition']) - if m: - disp = m.groupdict() - self.logDebug(disp) - if not disp['enc']: disp['enc'] = 'utf-8' - name = remove_chars(disp['name'], "\"';").strip() - name = unicode(unquote(name), disp['enc']) - - if not name: name = url - pyfile.name = name - self.logDebug("Filename: %s" % pyfile.name) - self.download(url, disposition=True)
\ No newline at end of file + pyfile.name = html_unescape(urlparse(pyfile.url).path.split("/")[-1]) + self.download(pyfile.url, disposition=True) diff --git a/module/plugins/hoster/BezvadataCz.py b/module/plugins/hoster/BezvadataCz.py index 680bbc173..a0717ad64 100644 --- a/module/plugins/hoster/BezvadataCz.py +++ b/module/plugins/hoster/BezvadataCz.py @@ -23,7 +23,7 @@ class BezvadataCz(SimpleHoster): __name__ = "BezvadataCz" __type__ = "hoster" __pattern__ = r"http://(\w*\.)*bezvadata.cz/stahnout/.*" - __version__ = "0.22" + __version__ = "0.23" __description__ = """BezvaData.cz""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") diff --git a/module/plugins/hoster/DlFreeFr.py b/module/plugins/hoster/DlFreeFr.py index 138d0e056..7cb58e6f4 100644 --- a/module/plugins/hoster/DlFreeFr.py +++ b/module/plugins/hoster/DlFreeFr.py @@ -19,7 +19,6 @@ class DlFreeFr(SimpleHoster): FILE_URL_PATTERN = r'href="(?P<url>http://.*?)">Télécharger ce fichier' def setup(self): - self.multiDL = True self.limitDL = 5 self.resumeDownload = True self.chunkLimit = 1 diff --git a/module/plugins/hoster/EasybytezCom.py b/module/plugins/hoster/EasybytezCom.py index 4fbd08756..0b46acb83 100644 --- a/module/plugins/hoster/EasybytezCom.py +++ b/module/plugins/hoster/EasybytezCom.py @@ -23,8 +23,8 @@ from random import random class EasybytezCom(SimpleHoster): __name__ = "EasybytezCom" __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?easybytez.com/(\w+).*" - __version__ = "0.05" + __pattern__ = r"http://(?:\w*\.)?easybytez.com/(\w{6,}).*" + __version__ = "0.06" __description__ = """easybytez.com""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") diff --git a/module/plugins/hoster/EuroshareEu.py b/module/plugins/hoster/EuroshareEu.py index a0bfe0ab2..1e1cc0b4b 100644 --- a/module/plugins/hoster/EuroshareEu.py +++ b/module/plugins/hoster/EuroshareEu.py @@ -37,7 +37,7 @@ class EuroshareEu(Hoster): __name__ = "EuroshareEu" __type__ = "hoster" __pattern__ = r"http://(\w*\.)?euroshare.eu/file/.*" - __version__ = "0.2b" + __version__ = "0.30" __description__ = """Euroshare.eu""" __author_name__ = ("zoidberg") diff --git a/module/plugins/hoster/FilesMailRu.py b/module/plugins/hoster/FilesMailRu.py index 6002ab3dc..1284329b5 100644 --- a/module/plugins/hoster/FilesMailRu.py +++ b/module/plugins/hoster/FilesMailRu.py @@ -2,9 +2,8 @@ # -*- coding: utf-8 -*- import re -from module.plugins.Hoster import Hoster +from module.plugins.Hoster import Hoster, chunks from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks def getInfo(urls): result = [] diff --git a/module/plugins/hoster/FilesonicCom.py b/module/plugins/hoster/FilesonicCom.py index 2788e7c62..f0fbab1d9 100644 --- a/module/plugins/hoster/FilesonicCom.py +++ b/module/plugins/hoster/FilesonicCom.py @@ -7,7 +7,7 @@ from urllib import unquote from module.plugins.Hoster import Hoster
from module.plugins.ReCaptcha import ReCaptcha
-from module.plugins.Plugin import chunks
+from module.utils import chunks
from module.network.RequestFactory import getURL
from module.common.json_layer import json_loads
diff --git a/module/plugins/hoster/MultishareCz.py b/module/plugins/hoster/MultishareCz.py index a0dda30b8..af7aa94cf 100644 --- a/module/plugins/hoster/MultishareCz.py +++ b/module/plugins/hoster/MultishareCz.py @@ -24,7 +24,7 @@ class MultishareCz(SimpleHoster): __name__ = "MultishareCz" __type__ = "hoster" __pattern__ = r"http://(?:\w*\.)?multishare.cz/stahnout/(?P<ID>\d+).*" - __version__ = "0.34" + __version__ = "0.40" __description__ = """MultiShare.cz""" __author_name__ = ("zoidberg") @@ -50,11 +50,12 @@ class MultishareCz(SimpleHoster): self.download("http://www.multishare.cz/html/download_free.php?ID=%s" % self.fileID) def handlePremium(self): - if not self.checkCredit(): + if not self.checkTrafficLeft(): self.logWarning("Not enough credit left to download file") self.resetAccount() self.download("http://www.multishare.cz/html/download_premium.php?ID=%s" % self.fileID) + self.checkTrafficLeft() def handleOverriden(self): if not self.premium: @@ -63,18 +64,13 @@ class MultishareCz(SimpleHoster): self.html = self.load('http://www.multishare.cz/html/mms_ajax.php', post = {"link": self.pyfile.url}, decode = True) self.getFileInfo() - if not self.checkCredit(): + if not self.checkTrafficLeft(): self.fail("Not enough credit left to download file") url = "http://dl%d.mms.multishare.cz/html/mms_process.php" % round(random()*10000*random()) params = {"u_ID" : self.acc_info["u_ID"], "u_hash" : self.acc_info["u_hash"], "link" : self.pyfile.url} self.logDebug(url, params) self.download(url, get = params) - - def checkCredit(self): - self.acc_info = self.account.getAccountInfo(self.user, True) - self.logInfo("User %s has %i MB left" % (self.user, self.acc_info["trafficleft"]/1024)) - - return self.pyfile.size / 1024 <= self.acc_info["trafficleft"] + self.checkTrafficLeft() getInfo = create_getInfo(MultishareCz)
\ No newline at end of file diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py index b2bec873d..382328496 100644 --- a/module/plugins/hoster/NetloadIn.py +++ b/module/plugins/hoster/NetloadIn.py @@ -5,11 +5,9 @@ import re from time import sleep, time +from module.utils import chunks from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks - - def getInfo(urls): ## returns list of tupels (name, size (in bytes), status (see FileDatabase), url) diff --git a/module/plugins/hoster/OronCom.py b/module/plugins/hoster/OronCom.py index 2b9f42798..120aa8ff4 100755 --- a/module/plugins/hoster/OronCom.py +++ b/module/plugins/hoster/OronCom.py @@ -15,7 +15,7 @@ def getInfo(urls): result.append((url, 0, 1, url)) continue - m = re.search(OronCom.FILE_INFO_PATTERN, html, re.MULTILINE) + m = re.search(OronCom.FILE_INFO_PATTERN, html) if m: name = m.group(1) hSize = float(m.group(2).replace(",", ".")) @@ -32,11 +32,11 @@ def getInfo(urls): class OronCom(Hoster): __name__ = "OronCom" __type__ = "hoster" - __pattern__ = r"http://(?:www.)?oron.com/(?!folder)\w+" - __version__ = "0.15" - __description__ = "Oron.com Hoster Plugin" + __pattern__ = r"http://(?:www.)?oron.com/" + __version__ = "0.13" + __description__ = "File Hoster: Oron.com" __author_name__ = ("chrox", "DHMH") - __author_mail__ = ("chrox@pyload.org", "webmaster@pcProfil.de") + __author_mail__ = ("chrox@pyload.org", "DHMH@pyload.org") FILE_INFO_PATTERN = r'(?:Filename|Dateiname): <b class="f_arial f_14px">(.*?)</b>\s*<br>\s*(?:Größe|File size): ([0-9,\.]+) (Kb|Mb|Gb)' @@ -129,13 +129,13 @@ class OronCom(Hoster): self.logError("error in parsing site") def handlePremium(self): - info = self.account.getAccountInfo(self.user, True) - self.logDebug("Traffic left: %s" % info['trafficleft']) + self.account.getAccountInfo(True) + self.logDebug("Traffic left: %s" % self.account.trafficleft) self.logDebug("File Size: %d" % int(self.pyfile.size / 1024)) - if int(self.pyfile.size / 1024) > info["trafficleft"]: + if int(self.pyfile.size / 1024) > self.account.trafficleft: self.logInfo(_("Not enough traffic left")) - self.account.empty(self.user) + self.account.empty() self.fail(_("Traffic exceeded")) post_url = "http://oron.com/" + self.file_id @@ -147,3 +147,4 @@ class OronCom(Hoster): self.html = self.load(post_url, post=post_dict, ref=False, decode=True).encode("utf-8") link = re.search('href="(.*?)" class="atitle"', self.html).group(1) self.download(link) + diff --git a/module/plugins/hoster/Premium4Me.py b/module/plugins/hoster/Premium4Me.py index d029b3df1..cd47a9e91 100644 --- a/module/plugins/hoster/Premium4Me.py +++ b/module/plugins/hoster/Premium4Me.py @@ -6,7 +6,7 @@ from module.plugins.Hoster import Hoster class Premium4Me(Hoster):
__name__ = "Premium4Me"
- __version__ = "0.03"
+ __version__ = "0.10"
__type__ = "hoster"
__pattern__ = r"http://premium4.me/.*"
diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py index 0d927c525..a4a72eb53 100644 --- a/module/plugins/hoster/RapidshareCom.py +++ b/module/plugins/hoster/RapidshareCom.py @@ -52,7 +52,7 @@ class RapidshareCom(Hoster): __pattern__ = r"https?://[\w\.]*?rapidshare.com/(?:files/(?P<id>\d*?)/(?P<name>[^?]+)|#!download\|(?:\w+)\|(?P<id_new>\d+)\|(?P<name_new>[^|]+))" __version__ = "1.37" __description__ = """Rapidshare.com Download Hoster""" - __config__ = [["server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None"]] + __config__ = [("server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None")] __author_name__ = ("spoob", "RaNaN", "mkaay") __author_mail__ = ("spoob@pyload.org", "ranan@pyload.org", "mkaay@mkaay.de") @@ -132,7 +132,7 @@ class RapidshareCom(Hoster): self.handleFree() def handlePremium(self): - info = self.account.getAccountInfo(self.user, True) + info = self.account.getAccountInfo(True) self.log.debug("%s: Use Premium Account" % self.__name__) url = self.api_data["mirror"] self.download(url, get={"directstart":1}) diff --git a/module/plugins/hoster/RealdebridCom.py b/module/plugins/hoster/RealdebridCom.py index 4aed32614..3a3ac99b0 100644 --- a/module/plugins/hoster/RealdebridCom.py +++ b/module/plugins/hoster/RealdebridCom.py @@ -5,15 +5,15 @@ import re from urllib import quote, unquote
from random import randrange
+from module.utils import encode
from module.plugins.Hoster import Hoster
class RealdebridCom(Hoster):
- __name__ = "RealdebridCom"
- __version__ = "0.42"
- __type__ = "hoster"
-
+ __version__ = "0.41"
__pattern__ = r"https?://.*real-debrid\..*"
__description__ = """Real-Debrid.com hoster plugin"""
+ __config__ = [("https", "bool", _("Enable HTTPS"), False)]
+
__author_name__ = ("Devirex, Hazzard")
__author_mail__ = ("naibaf_11@yahoo.de")
@@ -44,11 +44,14 @@ class RealdebridCom(Hoster): password = self.getPassword().splitlines()
if not password: password = ""
else: password = password[0]
-
- url = "http://real-debrid.com/lib/ajax/generator.php?lang=en&sl=1&link=%s&passwort=%s" % (quote(pyfile.url, ""), password)
+
+ url = "http://real-debrid.com/ajax/deb.php?lang=en&sl=1&link=%s&passwort=%s" % (quote(encode(pyfile.url), ""), password)
page = self.load(url)
error = re.search(r'<span id="generation-error">(.*)</span>', page)
+ generation_ok = re.search(r'<span id="generation-ok"><a href="(.*)">(.*)</a></span>', page)
+ if generation_ok:
+ page = generation_ok.group(1).strip()
if error:
msg = error.group(1).strip()
@@ -59,6 +62,9 @@ class RealdebridCom(Hoster): self.fail(msg)
elif url == 'error':
self.fail("Your IP is most likely blocked. Please contact RealDebrid support")
+ elif page == "File's hoster is in maintenance. Try again later.":
+ self.logWarning(page)
+ self.tempOffline()
else:
new_url = page
diff --git a/module/plugins/hoster/ShareonlineBiz.py b/module/plugins/hoster/ShareonlineBiz.py index d355eeffe..2d1fc8d85 100644 --- a/module/plugins/hoster/ShareonlineBiz.py +++ b/module/plugins/hoster/ShareonlineBiz.py @@ -7,10 +7,8 @@ import hashlib import random from time import sleep -from module.plugins.Hoster import Hoster +from module.plugins.Hoster import Hoster, chunks from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks - def getInfo(urls): api_url_base = "http://api.share-online.biz/linkcheck.php" @@ -52,7 +50,7 @@ class ShareonlineBiz(Hoster): self.multiDL = False self.chunkLimit = 1 - if self.account and self.account.isPremium(self.user): + if self.premium: self.multiDL = True def process(self, pyfile): @@ -60,7 +58,7 @@ class ShareonlineBiz(Hoster): pyfile.name = self.api_data["filename"] pyfile.sync() - if self.account and self.account.isPremium(self.user): + if self.premium: self.handleAPIPremium() #self.handleWebsitePremium() else: @@ -127,7 +125,7 @@ class ShareonlineBiz(Hoster): def handleAPIPremium(self): #should be working better self.resumeDownload = True - info = self.account.getUserAPI(self.user, self.req) + info = self.account.getUserAPI(self.req) if info["dl"].lower() == "not_available": self.fail("DL API error") self.req.cj.setCookie("share-online.biz", "dl", info["dl"]) diff --git a/module/plugins/hoster/TurbouploadCom.py b/module/plugins/hoster/TurbouploadCom.py index 59939d3c7..5fd81fb69 100644 --- a/module/plugins/hoster/TurbouploadCom.py +++ b/module/plugins/hoster/TurbouploadCom.py @@ -23,8 +23,8 @@ from module.plugins.hoster.EasybytezCom import EasybytezCom class TurbouploadCom(EasybytezCom): __name__ = "TurbouploadCom" __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?turboupload.com/(\w+).*" - __version__ = "0.01" + __pattern__ = r"http://(?:\w*\.)?turboupload.com/(\w{6,}).*" + __version__ = "0.02" __description__ = """turboupload.com""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") diff --git a/module/plugins/hoster/UploadedTo.py b/module/plugins/hoster/UploadedTo.py index 39483cf86..751dcda25 100644 --- a/module/plugins/hoster/UploadedTo.py +++ b/module/plugins/hoster/UploadedTo.py @@ -2,11 +2,10 @@ import re -from module.utils import html_unescape, parseFileSize +from module.utils import html_unescape, parseFileSize, chunks from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks from module.plugins.ReCaptcha import ReCaptcha key = "bGhGMkllZXByd2VEZnU5Y2NXbHhYVlZ5cEE1bkEzRUw=".decode('base64') diff --git a/module/plugins/hoster/WuploadCom.py b/module/plugins/hoster/WuploadCom.py index 11b61ae59..ffb082cbe 100644 --- a/module/plugins/hoster/WuploadCom.py +++ b/module/plugins/hoster/WuploadCom.py @@ -3,11 +3,12 @@ import re import string +from urllib import unquote from types import MethodType from module.plugins.Hoster import Hoster -from module.plugins.Plugin import chunks +from module.utils import chunks from module.network.RequestFactory import getURL from module.common.json_layer import json_loads @@ -31,7 +32,7 @@ def getInfo(urls): if item["status"] != "AVAILABLE": result.append((None, 0, 1, ids[str(item["id"])])) else: - result.append((item["filename"], item["size"], 2, ids[str(item["id"])])) + result.append((unquote(item["filename"]), item["size"], 2, ids[str(item["id"])])) yield result @@ -47,7 +48,7 @@ class WuploadCom(Hoster): __name__ = "WuploadCom" __type__ = "hoster" __pattern__ = r"http://[\w\.]*?wupload\..*?/file/(([a-z][0-9]+/)?[0-9]+)(/.*)?" - __version__ = "0.1" + __version__ = "0.2" __description__ = """Wupload com""" __author_name__ = ("jeix", "paulking") __author_mail__ = ("jeix@hasnomail.de", "") diff --git a/module/plugins/hoster/YoutubeCom.py b/module/plugins/hoster/YoutubeCom.py index 2b3ea7ed7..b6ea36a3c 100644 --- a/module/plugins/hoster/YoutubeCom.py +++ b/module/plugins/hoster/YoutubeCom.py @@ -75,10 +75,10 @@ class YoutubeCom(Hoster): fmt_dict[fmt] = unquote(url) self.logDebug("Found links: %s" % fmt_dict) - for fmt in fmt_dict.keys(): + for fmt in fmt_dict.keys(): if fmt not in self.formats: - self.logDebug("FMT not supported: %s" % fmt) - del fmt_dict[fmt] + self.logDebug("FMT not supported: %s" % fmt) + del fmt_dict[fmt] allowed = lambda x: self.getConfig(self.formats[x][0]) sel = lambda x: self.formats[x][3] #select quality index diff --git a/module/plugins/hoster/ZeveraCom.py b/module/plugins/hoster/ZeveraCom.py new file mode 100644 index 000000000..d1fa80802 --- /dev/null +++ b/module/plugins/hoster/ZeveraCom.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from module.plugins.Hoster import Hoster
+from module.utils import html_unescape
+from urllib import quote, unquote
+from time import sleep
+
+class ZeveraCom(Hoster):
+ __name__ = "ZeveraCom"
+ __version__ = "0.11"
+ __type__ = "hoster"
+ __pattern__ = r"http://zevera.com/.*"
+ __description__ = """zevera.com hoster plugin"""
+ __author_name__ = ("zoidberg")
+ __author_mail__ = ("zoidberg@mujmail.cz")
+
+ api_url = "http://zevera.com/API.ashx"
+
+ def process(self, pyfile):
+ if not self.account:
+ self.logError(_("Please enter your zevera.com account or deactivate this plugin"))
+ self.fail("No zevera.com account provided")
+
+ self.logDebug("zevera.com: Old URL: %s" % pyfile.url)
+
+ last_size = retries = 0
+ olink = self.pyfile.url #quote(self.pyfile.url.encode('utf_8'))
+
+ for i in range(100):
+ self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_request', olink = olink)
+ self.checkAPIErrors(self.retData)
+
+ if self.retData['FileInfo']['StatusID'] == 100:
+ break
+ elif self.retData['FileInfo']['StatusID'] == 99:
+ self.fail('Failed to initialize download (99)')
+ else:
+ if self.retData['FileInfo']['Progress']['BytesReceived'] <= last_size:
+ if retries >= 6:
+ self.fail('Failed to initialize download (%d)' % self.retData['FileInfo']['StatusID'] )
+ retries += 1
+ else:
+ retries = 0
+
+ last_size = self.retData['FileInfo']['Progress']['BytesReceived']
+
+ pyfile.progress = self.retData['FileInfo']['Progress']['Percentage']
+
+ self.setWait(self.retData['Update_Wait'])
+ self.wait()
+
+ pyfile.progress = 0
+ pyfile.name = self.crazyDecode(self.retData['FileInfo']['RealFileName'])
+ pyfile.size = self.retData['FileInfo']['FileSizeInBytes']
+
+ self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_start', FileID = self.retData['FileInfo']['FileID'])
+ self.checkAPIErrors(self.retData)
+
+ self.download(self.api_url, get = {
+ 'cmd': "open_stream",
+ 'login': self.account.loginname,
+ 'pass': self.account.password,
+ 'FileID': self.retData['FileInfo']['FileID'],
+ 'startBytes': 0
+ }
+ )
+
+ def checkAPIErrors(self, retData):
+ if not retData:
+ self.fail('Unknown API response')
+
+ if retData['ErrorCode']:
+ self.logError(retData['ErrorCode'], retData['ErrorMessage'])
+ self.fail('ERROR: ' + retData['ErrorMessage'])
+
+ if self.pyfile.size / 1024000 > retData['AccountInfo']['AvailableTODAYTrafficForUseInMBytes']:
+ self.logWarning("Not enough data left to download the file")
+
+ def crazyDecode(self, ustring):
+ # accepts decoded ie. unicode string - API response is double-quoted, double-utf8-encoded
+ # no idea what the proper order of calling these functions would be :-/
+ return html_unescape(unquote(unquote(ustring.replace('@DELIMITER@','#'))).encode('raw_unicode_escape').decode('utf-8'))
\ No newline at end of file diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py deleted file mode 100644 index d50df3943..000000000 --- a/module/plugins/internal/MultiHoster.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.utils import remove_chars -from module.plugins.Hook import Hook - -class MultiHoster(Hook): - """ - Generic MultiHoster plugin - """ - - interval = 0 - hosters = [] - replacements = [] - supported = [] - - def getHosterCached(self): - if not self.hosters: - - try: - self.hosters = self.getHoster() - except Exception, e: - self.logError("%s" % str(e)) - return [] - - for rep in self.replacements: - if rep[0] in self.hosters: - self.hosters.remove(rep[0]) - if rep[1] not in self.hosters: - self.hosters.append(rep[1]) - - return self.hosters - - - def getHoster(self): - """Load list of supported hoster - - :return: List of domain names - """ - raise NotImplementedError - - def coreReady(self): - pluginMap = {} - for name in self.core.pluginManager.hosterPlugins.keys(): - pluginMap[name.lower()] = name - - new_supported = [] - - for hoster in self.getHosterCached(): - name = remove_chars(hoster.lower(), "-.") - - if name in pluginMap: - self.supported.append(pluginMap[name]) - else: - new_supported.append(hoster) - - if not self.supported and not new_supported: - self.logError(_("No Hoster loaded")) - return - - module = self.core.pluginManager.getPlugin(self.__name__) - klass = getattr(module, self.__name__) - - # inject plugin plugin - self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) - for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - dict["new_module"] = module - dict["new_name"] = self.__name__ - - self.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) - - # create new regexp - regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) - - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict["pattern"] = regexp - dict["re"] = re.compile(regexp) - - - def unload(self): - for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - if "module" in dict: - del dict["module"] - - del dict["new_module"] - del dict["new_name"]
\ No newline at end of file diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index de6215704..a315fbea3 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -19,11 +19,10 @@ import os import re -from os.path import join from glob import glob from subprocess import Popen, PIPE -from module.utils import save_join, decode +from module.utils.fs import save_join, decode, fs_encode from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError class UnRar(AbtractExtractor): @@ -39,7 +38,7 @@ class UnRar(AbtractExtractor): @staticmethod def checkDeps(): if os.name == "nt": - UnRar.CMD = join(pypath, "UnRAR.exe") + UnRar.CMD = save_join(pypath, "UnRAR.exe") p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) p.communicate() else: @@ -80,7 +79,7 @@ class UnRar(AbtractExtractor): self.password = "" #save the correct password def checkArchive(self): - p = self.call_unrar("l", "-v", self.file) + p = self.call_unrar("l", "-v", fs_encode(self.file)) out, err = p.communicate() if self.re_wrongpwd.search(err): self.passwordProtected = True @@ -102,7 +101,7 @@ class UnRar(AbtractExtractor): def checkPassword(self, password): #at this point we can only verify header protected files if self.headerProtected: - p = self.call_unrar("l", "-v", self.file, password=password) + p = self.call_unrar("l", "-v", fs_encode(self.file), password=password) out, err = p.communicate() if self.re_wrongpwd.search(err): return False @@ -115,7 +114,7 @@ class UnRar(AbtractExtractor): # popen thinks process is still alive (just like pexpect) - very strange behavior # so for now progress can not be determined correctly - p = self.call_unrar(command, self.file, self.out, password=password) + p = self.call_unrar(command, fs_encode(self.file), self.out, password=password) renice(p.pid, self.renice) progress(0) @@ -141,7 +140,7 @@ class UnRar(AbtractExtractor): def listContent(self): command = "vb" if self.fullpath else "lb" - p = self.call_unrar(command, "-v", self.file, password=self.password) + p = self.call_unrar(command, "-v", fs_encode(self.file), password=self.password) out, err = p.communicate() if "Cannot open" in err: @@ -176,7 +175,7 @@ class UnRar(AbtractExtractor): #NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) - self.m.logDebug(" ".join(call)) + self.m.logDebug(" ".join([decode(arg) for arg in call])) p = Popen(call, stdout=PIPE, stderr=PIPE) |