diff options
23 files changed, 320 insertions, 227 deletions
diff --git a/module/plugins/crypter/EmbeduploadCom.py b/module/plugins/crypter/EmbeduploadCom.py index e4a9387ae..957e1efb2 100644 --- a/module/plugins/crypter/EmbeduploadCom.py +++ b/module/plugins/crypter/EmbeduploadCom.py @@ -55,7 +55,7 @@ class EmbeduploadCom(Crypter): try: header = self.load(link, just_header=True) if 'location' in header: - new_links.append(header['location']) + new_links.append(header.get('location')) except BadHeader: pass return new_links diff --git a/module/plugins/crypter/SafelinkingNet.py b/module/plugins/crypter/SafelinkingNet.py index c5fac837a..a11ae2cd8 100644 --- a/module/plugins/crypter/SafelinkingNet.py +++ b/module/plugins/crypter/SafelinkingNet.py @@ -34,7 +34,7 @@ class SafelinkingNet(Crypter): header = self.load(url, just_header=True) if 'location' in header: - self.urls = [header['location']] + self.urls = [header.get('location')] else: self.error(_("Couldn't find forwarded Link")) diff --git a/module/plugins/crypter/ShSt.py b/module/plugins/crypter/ShSt.py index 43cc4e779..429d3f3e2 100644 --- a/module/plugins/crypter/ShSt.py +++ b/module/plugins/crypter/ShSt.py @@ -27,5 +27,5 @@ class ShSt(Crypter): self.req.http.c.setopt(pycurl.USERAGENT, "curl/7.42.1") #: Fetch the target URL header = self.load(self.pyfile.url, just_header = True, decode = False) - target_url = header['location'] + target_url = header.get('location') self.urls.append(target_url) diff --git a/module/plugins/crypter/XupPl.py b/module/plugins/crypter/XupPl.py index ac918a6de..458372d13 100644 --- a/module/plugins/crypter/XupPl.py +++ b/module/plugins/crypter/XupPl.py @@ -21,6 +21,6 @@ class XupPl(Crypter): def decrypt(self, pyfile): header = self.load(pyfile.url, just_header=True) if 'location' in header: - self.urls = [header['location']] + self.urls = [header.get('location')] else: self.fail(_("Unable to find link")) diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 47ed708be..034a4b81a 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -63,23 +63,15 @@ class ArchiveQueue(object): def get(self): - try: - return [int(pid) for pid in self.plugin.retrieve("ExtractArchive:%s" % self.storage, "").decode('base64').split()] - - except Exception: - return [] + return self.plugin.retrieve(self.storage, default=[]) def set(self, value): - if isinstance(value, list): - item = str(value)[1:-1].replace(' ', '').replace(',', ' ') - else: - item = str(value).strip() - return self.plugin.store("ExtractArchive:%s" % self.storage, item.encode('base64')[:-1]) + return self.plugin.store(self.storage, value) def delete(self): - return self.plugin.delete("ExtractArchive:%s" % self.storage) + return self.plugin.delete(self.storage) def add(self, item): @@ -107,7 +99,7 @@ class ArchiveQueue(object): class ExtractArchive(Addon): __name__ = "ExtractArchive" __type__ = "hook" - __version__ = "1.51" + __version__ = "1.52" __status__ = "testing" __config__ = [("activated" , "bool" , "Activated" , True ), diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py index fdaa928eb..34b992d5b 100644 --- a/module/plugins/hooks/UpdateManager.py +++ b/module/plugins/hooks/UpdateManager.py @@ -16,7 +16,7 @@ from module.utils import fs_encode, save_join as fs_join class UpdateManager(Addon): __name__ = "UpdateManager" __type__ = "hook" - __version__ = "0.58" + __version__ = "1.00" __status__ = "testing" __config__ = [("activated" , "bool", "Activated" , True ), @@ -32,7 +32,10 @@ class UpdateManager(Addon): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - SERVER_URL = "http://updatemanager.pyload.org" + _VERSION = re.compile(r'__version__.*=.*("|\')([\d.]+)') + + SERVER_URL = "http://updatemanager.pyload.org" + PERIODICAL_INTERVAL = 3 * 60 * 60 #: 3 hours @@ -40,6 +43,7 @@ class UpdateManager(Addon): if self.checkonstart: self.pyload.api.pauseServer() self.update() + if self.do_restart is False: self.pyload.api.unpauseServer() @@ -47,7 +51,7 @@ class UpdateManager(Addon): def init(self): - self.info = {'pyload': False, 'version': None, 'plugins': False, 'last_check': time.time()} + self.info = {'pyload': False, 'plugins': False, 'last_check': time.time()} self.mtimes = {} #: Store modification time for each plugin self.event_map = {'allDownloadsProcessed': "all_downloads_processed"} @@ -93,14 +97,12 @@ class UpdateManager(Addon): """ Reload and reindex all modified plugins """ + reloads = [] modules = filter( lambda m: m and (m.__name__.startswith("module.plugins.") or m.__name__.startswith("userplugins.")) and m.__name__.count(".") >= 2, sys.modules.values() ) - - reloads = [] - for m in modules: root, type, name = m.__name__.rsplit(".", 2) id = (type, name) @@ -113,6 +115,7 @@ class UpdateManager(Addon): if id not in self.mtimes: self.mtimes[id] = mtime + elif self.mtimes[id] < mtime: reloads.append(id) self.mtimes[id] = mtime @@ -120,14 +123,25 @@ class UpdateManager(Addon): return True if self.pyload.pluginManager.reloadPlugins(reloads) else False - def server_response(self): + def server_response(self, line=None): try: - return self.load(self.SERVER_URL, - get={'v': self.pyload.api.getServerVersion()}).splitlines() + html = self.load(self.SERVER_URL, + get={'v': self.pyload.api.getServerVersion()}) except Exception: self.log_warning(_("Unable to retrieve server to get updates")) + else: + res = html.splitlines() + + if line is not None: + try: + res = res[line] + except IndexError: + res = None + + return res + @Expose @threaded @@ -135,35 +149,34 @@ class UpdateManager(Addon): """ Check for updates """ - if self._update() == 2 and self.get_config('autorestart'): - if not self.pyload.api.statusDownloads(): - self.pyload.api.restart() - else: - self.do_restart = True - self.log_warning(_("Downloads are active, will restart once the download is done")) - self.pyload.api.pauseServer() + if self._update() is not 2 or not self.get_config('autorestart'): + return + + if not self.pyload.api.statusDownloads(): + self.pyload.api.restart() + else: + self.do_restart = True + self.log_warning(_("Downloads are active, will restart once the download is done")) + self.pyload.api.pauseServer() def _update(self): - data = self.server_response() + newversion = self.server_response(0) + self.info['pyload'] = False self.info['last_check'] = time.time() - if not data: + if not newversion: exitcode = 0 - elif data[0] == "None": + elif newversion == "None": self.log_info(_("No new pyLoad version available")) - exitcode = self._update_plugins(data[1:]) - - elif onlyplugin: - exitcode = 0 + exitcode = self.update_plugins() else: - self.log_info(_("*** New pyLoad Version %s available ***") % data[0]) + self.log_info(_("*** New pyLoad Version %s available ***") % newversion) self.log_info(_("*** Get it here: https://github.com/pyload/pyload/releases ***")) - self.info['pyload'] = True - self.info['version'] = data[0] + self.info['pyload'] = True exitcode = 3 #: Exit codes: @@ -174,64 +187,101 @@ class UpdateManager(Addon): return exitcode - def _update_plugins(self, data): - """ - Check for plugin updates - """ - exitcode = 0 - updated = [] + @Expose + def update_plugins(self): + updates = self.server_response() + + if not updates or updates[0] != "None": + return 0 - url = data[0] - schema = data[1].split('|') + updated = self._update_plugins(updates) + + if updated: + self.log_info(_("*** Plugins updated ***")) - VERSION = re.compile(r'__version__.*=.*("|\')([\d.]+)') + if self.pyload.pluginManager.reloadPlugins(updated): + exitcode = 1 + else: + self.log_warning(_("You have to restart pyLoad to reload the updated plugins")) + self.info['plugins'] = True + exitcode = 2 - if "BLACKLIST" in data: - blacklist = data[data.index('BLACKLIST') + 1:] - updatelist = data[2:data.index('BLACKLIST')] + self.manager.dispatchEvent("plugin_updated", updated) + else: + self.log_info(_("*** No plugin updates available ***")) + exitcode = 0 + + #: Exit codes: + #: 0 = No plugin updated + #: 1 = Plugins updated + #: 2 = Plugins updated, but restart required + return exitcode + + + def parse_list(self, list): + schema = list[2].split('|') + + if "BLACKLIST" in list: + blacklist = list[list.index('BLACKLIST') + 1:] + updatelist = list[3:list.index('BLACKLIST')] else: blacklist = [] - updatelist = data[2:] + updatelist = list[3:] - updatelist = [dict(zip(schema, x.split('|'))) for x in updatelist] - blacklist = [dict(zip(schema, x.split('|'))) for x in blacklist] + for l in updatelist, blacklist: + nl = [] + for line in l: + d = dict(zip(schema, line.split('|'))) + d['name'] = d['name'].rsplit('.py', 1)[0] + d['type'] = d['type'].rstrip('s') + nl.append(d) + l[:] = nl - if blacklist: - type_plugins = [(plugin['type'], plugin['name'].rsplit('.', 1)[0]) for plugin in blacklist] + updatelist = sorted(updatelist, key=operator.itemgetter("type", "name")) + blacklist = sorted(blacklist, key=operator.itemgetter("type", "name")) - #: Protect UpdateManager from self-removing - try: - type_plugins.remove(("hook", "UpdateManager")) - except ValueError: - pass + return updatelist, blacklist + + + def _update_plugins(self, updates): + """ + Check for plugin updates + """ + updated = [] + + updatelist, blacklist = self.parse_list(updates) + + url = updates[1] - for t, n in type_plugins: - for idx, plugin in enumerate(updatelist): - if n is plugin['name'] and t is plugin['type']: - updatelist.pop(idx) - break + if blacklist: + #@NOTE: Protect UpdateManager from self-removing + type_plugins = [(plugin['type'], plugin['name']) for plugin in blacklist \ + if plugin['name'] is not self.__name__ and plugin['type'] is not self.__type__] + + c = 1 + l = len(type_plugins) + for idx, plugin in enumerate(updatelist): + if c > l: + break + name = plugin['name'] + type = plugin['type'] + for t, n in type_plugins: + if n != name or t != type: + continue + updatelist.pop(idx) + c += 1 + break - for t, n in self.remove_plugins(sorted(type_plugins)): + for t, n in self.remove_plugins(type_plugins): self.log_info(_("Removed blacklisted plugin: %(type)s %(name)s") % { 'type': t.upper(), 'name': n, }) - for plugin in sorted(updatelist, key=operator.itemgetter("type", "name")): - filename = plugin['name'] - prefix = plugin['type'] - version = plugin['version'] - - if filename.endswith(".pyc"): - name = filename[:filename.find("_")] - else: - name = filename.replace(".py", "") - - #@TODO: Remove in 0.4.10 - if prefix.endswith("s"): - type = prefix[:-1] - else: - type = prefix + for plugin in updatelist: + name = plugin['name'] + type = plugin['type'] + version = plugin['version'] plugins = getattr(self.pyload.pluginManager, "%sPlugins" % type) @@ -239,50 +289,38 @@ class UpdateManager(Addon): newver = float(version) if not oldver: - msg = "New plugin: [%(type)s] %(name)s (v%(newver).2f)" + msg = "New plugin: %(type)s %(name)s (v%(newver).2f)" elif newver > oldver: - msg = "New version of plugin: [%(type)s] %(name)s (v%(oldver).2f -> v%(newver).2f)" + msg = "New version of plugin: %(type)s %(name)s (v%(oldver).2f -> v%(newver).2f)" else: continue - self.log_info(_(msg) % {'type' : type, - 'name' : name, - 'oldver': oldver, - 'newver': newver}) + self.log_info(_(msg) % {'type' : type.upper(), + 'name' : name, + 'oldver': oldver, + 'newver': newver}) try: - content = self.load(url % plugin, decode=False) - m = VERSION.search(content) + content = self.load(url % plugin + ".py", decode=False) + m = self._VERSION.search(content) if m and m.group(2) == version: - with open(fs_join("userplugins", prefix, filename), "wb") as f: + #@TODO: Remove in 0.4.10 + if type in ("account", "hook"): + folder = type + "s" + else: + folder = type + + with open(fs_join("userplugins", folder, name + ".py"), "wb") as f: f.write(fs_encode(content)) - updated.append((prefix, name)) + updated.append((type, name)) else: raise Exception(_("Version mismatch")) except Exception, e: - self.log_error(_("Error updating plugin: [%s] %s") % (type, name), e) + self.log_error(_("Error updating plugin: %s %s") % (type.upper(), name), e) - if updated: - self.log_info(_("*** Plugins updated ***")) - - if self.pyload.pluginManager.reloadPlugins(updated): - exitcode = 1 - else: - self.log_warning(_("You have to restart pyLoad to reload the updated plugins")) - self.info['plugins'] = True - exitcode = 2 - - self.manager.dispatchEvent("plugin_updated", updated) - else: - self.log_info(_("No plugin updates available")) - - #: Exit codes: - #: 0 = No plugin updated - #: 1 = Plugins updated - #: 2 = Plugins updated, but restart required - return exitcode + return updated #: Deprecated method, use `remove_plugins` instead @@ -309,16 +347,17 @@ class UpdateManager(Addon): for type, name in type_plugins: rootplugins = os.path.join(pypath, "module", "plugins") - for dir in ("userplugins", rootplugins): - py_filename = fs_join(dir, type, name + ".py") - - #@TODO: Remove in 0.4.10 - if type == "hook": - py_filename = fs_join(dir, "hooks" , name + ".py") + #@TODO: Remove in 0.4.10 + if type in ("account", "hook"): + folder = type + "s" + else: + folder = type + for dir in ("userplugins", rootplugins): + py_filename = fs_join(dir, folder, name + ".py") pyc_filename = py_filename + "c" - if type == "hook": + if type is "hook": try: self.manager.deactivateHook(name) diff --git a/module/plugins/hoster/BitshareCom.py b/module/plugins/hoster/BitshareCom.py index e426de092..5b9abe7cc 100644 --- a/module/plugins/hoster/BitshareCom.py +++ b/module/plugins/hoster/BitshareCom.py @@ -84,7 +84,7 @@ class BitshareCom(SimpleHoster): if self.premium: header = self.load(self.pyfile.url, just_header=True) if 'location' in header: - return header['location'] + return header.get('location') #: Get download info self.log_debug("Getting download info") diff --git a/module/plugins/hoster/DlFreeFr.py b/module/plugins/hoster/DlFreeFr.py index f77b9596a..6c094636e 100644 --- a/module/plugins/hoster/DlFreeFr.py +++ b/module/plugins/hoster/DlFreeFr.py @@ -125,12 +125,14 @@ class DlFreeFr(SimpleHoster): value = value.strip() if key in header: - if type(header[key]) is list: - header[key].append(value) + header_key = header.get(key) + if type(header_key) is list: + header_key.append(value) else: - header[key] = [header[key], value] + header[key] = [header_key, value] else: header[key] = value + return header diff --git a/module/plugins/hoster/FreeWayMe.py b/module/plugins/hoster/FreeWayMe.py index c0053fc76..501b544c6 100644 --- a/module/plugins/hoster/FreeWayMe.py +++ b/module/plugins/hoster/FreeWayMe.py @@ -38,13 +38,13 @@ class FreeWayMe(MultiHoster): just_header=True) if 'location' in header: - headers = self.load(header['location'], just_header=True) + headers = self.load(header.get('location'), just_header=True) if headers['code'] == 500: #: Error on 2nd stage self.log_error(_("Error [stage2]")) else: #: Seems to work.. - self.download(header['location']) + self.download(header.get('location')) break else: #: Error page first stage diff --git a/module/plugins/hoster/OneFichierCom.py b/module/plugins/hoster/OneFichierCom.py index a8e7f6dc8..6c1d4ab82 100644 --- a/module/plugins/hoster/OneFichierCom.py +++ b/module/plugins/hoster/OneFichierCom.py @@ -99,7 +99,7 @@ class OneFichierCom(SimpleHoster): def handle_free(self, pyfile): self.check_errors() - url, inputs = self.parse_html_form('action="https://1fichier.com/\?[a-zA-Z0-9]+') + url, inputs = self.parse_html_form('action="https://1fichier.com/\?[\w^_]+') if not url: return diff --git a/module/plugins/hoster/SmoozedCom.py b/module/plugins/hoster/SmoozedCom.py index e864bb2c0..29768d50b 100644 --- a/module/plugins/hoster/SmoozedCom.py +++ b/module/plugins/hoster/SmoozedCom.py @@ -59,4 +59,4 @@ class SmoozedCom(MultiHoster): if not "location" in header: self.fail(_("Unable to initialize download")) else: - self.link = header['location'][-1] if isinstance(header['location'], list) else header['location'] + self.link = header.get('location')[-1] if isinstance(header.get('location'), list) else header.get('location') diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index a0c46a6a1..a1ecdd76f 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -40,7 +40,7 @@ class Account(Plugin): #: Callback of periodical job task self.cb = None - self.interval = None + self.interval = self.PERIODICAL_INTERVAL self.init() diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 47c5584f5..bdde514c0 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -23,7 +23,7 @@ def threaded(fn): class Addon(Plugin): __name__ = "Addon" __type__ = "hook" #@TODO: Change to `addon` in 0.4.10 - __version__ = "0.11" + __version__ = "0.12" __status__ = "testing" __threaded__ = [] #@TODO: Remove in 0.4.10 @@ -53,7 +53,7 @@ class Addon(Plugin): #: Callback of periodical job task, used by HookManager self.cb = None - self.interval = None + self.interval = self.PERIODICAL_INTERVAL self.init() self.init_events() @@ -132,6 +132,14 @@ class Addon(Plugin): raise NotImplementedError + def save_info(self): + self.store("info", self.info) + + + def restore_info(self): + self.retrieve("info", self.info) + + @property def activated(self): """ @@ -154,6 +162,7 @@ class Addon(Plugin): #: Deprecated method, use `deactivate` instead (Remove in 0.4.10) def unload(self, *args, **kwargs): + self.save_info() return self.deactivate(*args, **kwargs) @@ -166,6 +175,7 @@ class Addon(Plugin): #: Deprecated method, use `activate` instead (Remove in 0.4.10) def coreReady(self, *args, **kwargs): + self.restore_info() return self.activate(*args, **kwargs) diff --git a/module/plugins/internal/Base.py b/module/plugins/internal/Base.py index 423e53695..e3aaf202b 100644 --- a/module/plugins/internal/Base.py +++ b/module/plugins/internal/Base.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -import re import inspect import mimetypes import os +import re import time import urlparse @@ -47,7 +47,7 @@ def check_abort(fn): class Base(Plugin): __name__ = "Base" __type__ = "base" - __version__ = "0.09" + __version__ = "0.11" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -64,6 +64,9 @@ class Base(Plugin): def __init__(self, pyfile): self._init(pyfile.m.core) + #: + self.premium = None + #: Engage wan reconnection self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10 @@ -95,6 +98,9 @@ class Base(Plugin): #: Dict of the amount of retries already made self.retries = {} + self.init_base() + self.init() + def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) @@ -124,6 +130,10 @@ class Base(Plugin): return info + def init_base(self): + pass + + def init(self): """ Initialize the plugin (in addition to `__init__`) @@ -131,6 +141,10 @@ class Base(Plugin): pass + def setup_base(self): + pass + + def setup(self): """ Setup for enviroment and other things, called before downloading (possibly more than one time) @@ -156,16 +170,13 @@ class Base(Plugin): pass if self.account: - self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) - self.chunk_limit = -1 #: -1 for unlimited - self.resume_download = True - self.premium = self.account.info['data']['premium'] #: Don't use `self.account.premium` to avoid one unnecessary get_info call + self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) + self.premium = self.account.info['data']['premium'] #@NOTE: Avoid one unnecessary get_info call by `self.account.premium` here else: - self.req = self.pyload.requestFactory.getRequest(self.__name__) - self.chunk_limit = 1 - self.resume_download = False - self.premium = False + self.req = self.pyload.requestFactory.getRequest(self.__name__) + self.premium = False + self.setup_base() self.setup() @@ -306,7 +317,7 @@ class Base(Plugin): if msg: self.pyfile.error = msg else: - msg = self.pyfile.error or (self.info['error'] if 'error' in self.info else self.pyfile.getStatusName()) + msg = self.pyfile.error or self.info.get('error') or self.pyfile.getStatusName() raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 @@ -464,10 +475,11 @@ class Base(Plugin): value = value.strip() if key in header: - if type(header[key]) is list: - header[key].append(value) + header_key = header.get(key) + if type(header_key) is list: + header_key.append(value) else: - header[key] = [header[key], value] + header[key] = [header_key, value] else: header[key] = value @@ -475,7 +487,7 @@ class Base(Plugin): link = url elif header.get('location'): - location = self.fixurl(header['location'], url) + location = self.fixurl(header.get('location'), url) if header.get('code') == 302: link = location @@ -488,7 +500,7 @@ class Base(Plugin): extension = os.path.splitext(parse_name(url))[-1] if header.get('content-type'): - mimetype = header['content-type'].split(';')[0].strip() + mimetype = header.get('content-type').split(';')[0].strip() elif extension: mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index db13e445c..20322bb33 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.Base import Base, parse_name +from module.plugins.internal.Base import Base, check_abort, create_getInfo, parse_fileInfo +from module.plugins.internal.Plugin import parse_name from module.utils import save_path as safe_filename class Crypter(Base): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.12" + __version__ = "0.13" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -20,22 +21,12 @@ class Crypter(Base): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - def __init__(self, *args, **kwargs): - super(Crypter, self).__init__(*args, **kwargs) + def init_base(self): + self.packages = [] #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) + self.urls = [] #: List of urls, pyLoad will generate packagenames - #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) - self.packages = [] - - #: List of urls, pyLoad will generate packagenames - self.urls = [] - - self._setup() - self.init() - - - def _setup(self): - super(Crypter, self)._setup() + def setup_base(self): self.packages = [] self.urls = [] diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index c65946413..d81154554 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -5,7 +5,7 @@ from __future__ import with_statement import os import re -from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo, parse_fileInfo +from module.plugins.internal.Base import Base, check_abort, create_getInfo, parse_fileInfo from module.plugins.internal.Plugin import Fail, Retry, encode, exists, fixurl, parse_name from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename @@ -13,7 +13,7 @@ from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path a class Hoster(Base): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.36" + __version__ = "0.37" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -26,11 +26,15 @@ class Hoster(Base): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - def __init__(self, *args, **kwargs): - super(Hoster, self).__init__(*args, **kwargs) - + def init_base(self): #: Enable simultaneous processing of multiple downloads - self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 + self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 + + #: + self.chunk_limit = None + + #: + self.resume_download = False #: Location where the last call to download was saved self.last_download = None @@ -41,17 +45,19 @@ class Hoster(Base): #: Restart flag self.rst_free = False #@TODO: Recheck in 0.4.10 - self._setup() - self.init() - - - def _setup(self): - super(Hoster, self)._setup() + def setup_base(self): self.last_download = None self.last_check = None self.rst_free = False + if self.account: + self.chunk_limit = -1 #: -1 for unlimited + self.resume_download = True + else: + self.chunk_limit = 1 + self.resume_download = False + def load_account(self): if self.rst_free: @@ -76,7 +82,8 @@ class Hoster(Base): self.pyfile.setStatus("starting") try: - self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) #@TODO: Remove in 0.4.10 + self.log_debug("PROCESS URL " + self.pyfile.url, + "PLUGIN VERSION %s" % self.__version__) #@TODO: Remove in 0.4.10 self.process(self.pyfile) self.check_abort() @@ -94,7 +101,7 @@ class Hoster(Base): @check_abort - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True, resume=None, chunks=None): """ Downloads the content at url to download folder @@ -141,10 +148,16 @@ class Hoster(Base): self.check_abort() + chunks = min(self.pyload.config.get("download", "chunks"), chunks or self.chunk_limit or -1) + + if resume is None: + resume = self.resume_download + try: - newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, - chunks=self.get_chunk_count(), resume=self.resume_download, - progressNotify=self.pyfile.setProgress, disposition=disposition) + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, + cookies=cookies, chunks=chunks, resume=resume, + progressNotify=self.pyfile.setProgress, + disposition=disposition) finally: self.pyfile.size = self.req.size @@ -159,7 +172,8 @@ class Hoster(Base): os.rename(oldname_enc, newname_enc) except OSError, e: - self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e) + self.log_warning(_("Error renaming `%s` to `%s`") + % (newname, finalname), e) finalname = newname self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname)) diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index f5c3d091d..231615ae5 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -104,8 +104,8 @@ class MultiHoster(SimpleHoster): self.log_warning(_("Premium download failed")) self.restart(premium=False) - elif self.get_config("revertfailed", True) \ - and "new_module" in self.pyload.pluginManager.hosterPlugins[self.__name__]: + elif self.get_config("revertfailed", True) and \ + self.pyload.pluginManager.hosterPlugins[self.__name__].get('new_module'): hdict = self.pyload.pluginManager.hosterPlugins[self.__name__] tmp_module = hdict['new_module'] diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index d6eac7e58..a2392fdcc 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -6,6 +6,7 @@ import datetime import inspect import os import re +import sys import time import traceback import urllib @@ -17,17 +18,38 @@ if os.name is not "nt": import grp import pwd +from module.common.json_layer import json_dumps, json_loads from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 -from module.utils import fs_encode, fs_decode, html_unescape, parseFileSize as parse_size, save_join as fs_join +from module.utils import (fs_encode, fs_decode, get_console_encoding, html_unescape, + parseFileSize as parse_size, save_join as fs_join) #@TODO: Move to utils in 0.4.10 -def decode(string, encoding='utf8'): - """ Decode string to unicode with utf8 """ +def isiterable(obj): + return hasattr(obj, "__iter__") + + +#@TODO: Move to utils in 0.4.10 +def decode(string, encoding=None): + """Encoded string (default to UTF-8) -> unicode string""" if type(string) is str: - return string.decode(encoding, "replace") + try: + res = unicode(string, encoding or "utf-8") + + except UnicodeDecodeError, e: + if encoding: + raise UnicodeDecodeError(e) + + encoding = get_console_encoding(sys.stdout.encoding) + res = unicode(string, encoding) + + elif type(string) is unicode: + res = string + else: - return unicode(string) + res = unicode(string) + + return res #@TODO: Remove in 0.4.10 @@ -36,12 +58,18 @@ def _decode(*args, **kwargs): #@TODO: Move to utils in 0.4.10 -def encode(string, encoding='utf8'): - """ Decode string to utf8 """ +def encode(string, encoding=None, decoding=None): + """Unicode or decoded string -> encoded string (default to UTF-8)""" if type(string) is unicode: - return string.encode(encoding, "replace") + res = string.encode(encoding or "utf-8") + + elif type(string) is str: + res = encode(decode(string, decoding), encoding) + else: - return str(string) + res = str(string) + + return res #@TODO: Move to utils in 0.4.10 @@ -186,11 +214,11 @@ def parse_html_tag_attr_value(attr_name, tag): def parse_html_form(attr_str, html, input_names={}): for form in re.finditer(r"(?P<TAG><form[^>]*%s[^>]*>)(?P<CONTENT>.*?)</?(form|body|html)[^>]*>" % attr_str, - html, re.S | re.I): + html, re.I | re.S): inputs = {} action = parse_html_tag_attr_value("action", form.group('TAG')) - for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.S | re.I): + for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.I | re.S): name = parse_html_tag_attr_value("name", inputtag.group(1)) if name: value = parse_html_tag_attr_value("value", inputtag.group(1)) @@ -234,7 +262,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.52" + __version__ = "0.53" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -301,7 +329,7 @@ class Plugin(object): def log_error(self, *args, **kwargs): self._log("error", self.__type__, self.__name__, args) - if kwargs.get('trace'): + if self.pyload.debug and kwargs.get('trace', True): print "Traceback (most recent call last):" traceback.print_stack() @@ -338,12 +366,6 @@ class Plugin(object): self.log_warning(_("Setting owner and group failed"), e) - def get_chunk_count(self): - if self.chunk_limit <= 0: - return self.pyload.config.get("download", "chunks") - return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) - - def set_config(self, option, value, plugin=None): """ Set config value for current plugin @@ -374,14 +396,26 @@ class Plugin(object): """ Saves a value persistently to the database """ - self.pyload.db.setStorage(self.__name__, key, value) + value = map(decode, value) if isiterable(value) else decode(value) + entry = json_dumps(value).encode('base64') + self.pyload.db.setStorage(self.__name__, key, entry) - def retrieve(self, key, default=None): + def retrieve(self, key=None, default=None): """ Retrieves saved value or dict of all saved entries if key is None """ - return self.pyload.db.getStorage(self.__name__, key) or default + entry = self.pyload.db.getStorage(self.__name__, key) + + if entry: + if type(entry) is dict: + value = dict((k, json_loads(v.decode('base64'))) for k, v in value.items()) or default + else: + value = json_loads(entry.decode('base64')) or default + else: + value = entry + + return value def delete(self, key): @@ -456,13 +490,13 @@ class Plugin(object): frame = inspect.currentframe() try: - framefile = fs_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + framefile = fs_join("tmp", self.__name__, "%s_line%s.dump.html" % + (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) if not exists(os.path.join("tmp", self.__name__)): os.makedirs(os.path.join("tmp", self.__name__)) with open(framefile, "wb") as f: - f.write(encode(html)) except IOError, e: @@ -487,10 +521,11 @@ class Plugin(object): value = value.strip() if key in header: - if type(header[key]) is list: - header[key].append(value) + header_key = header.get(key) + if type(header_key) is list: + header_key.append(value) else: - header[key] = [header[key], value] + header[key] = [header_key, value] else: header[key] = value diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index d4ecd3117..ba4235072 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.66" + __version__ = "0.67" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -82,20 +82,16 @@ class SimpleCrypter(Crypter, SimpleHoster): #@TODO: Remove in 0.4.10 def _setup(self): orig_name = self.__name__ - self.__name__ = (orig_name + ".py").replace("Folder.py", "").replace(".py", "") - + self.__name__ = re.sub(r'Folder$', "", self.__name__) super(SimpleCrypter, self)._setup() - self.__name__ = orig_name #@TODO: Remove in 0.4.10 def load_account(self): orig_name = self.__name__ - self.__name__ = (orig_name + ".py").replace("Folder.py", "").replace(".py", "") - + self.__name__ = re.sub(r'Folder$', "", self.__name__) super(SimpleCrypter, self).load_account() - self.__name__ = orig_name @@ -106,7 +102,7 @@ class SimpleCrypter(Crypter, SimpleHoster): header = self.load(redirect, just_header=True) if header.get('location'): - self.link = header['location'] + self.link = header.get('location') else: break else: diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 0f27ecc91..cf74c6646 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -270,7 +270,7 @@ class SimpleHoster(Hoster): if not self.link and not self.last_download: self.preload() - if 'status' not in self.info or self.info['status'] is 3: #@TODO: Recheck in 0.4.10 + if self.info.get('status', 3) is 3: #@TODO: Recheck in 0.4.10 self.check_info() if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index f31a386a7..6f85c286a 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -38,7 +38,7 @@ class UnRar(Extractor): re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?(\.rev|\.bad)?', re.I) re_filefixed = re.compile(r'Building (.+)') - re_filelist = re.compile(r'^(.)(\s*[\w\-.]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.M|re.I) + re_filelist = re.compile(r'^(.)(\s*[\w\-.]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.I | re.M) re_wrongpwd = re.compile(r'password', re.I) re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error|corrupt', re.I) diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index dd2fba568..7ec9f3895 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +import re + from module.plugins.internal.Plugin import set_cookie from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo @@ -7,7 +9,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.17" + __version__ = "0.18" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -49,7 +51,7 @@ class XFSCrypter(SimpleCrypter): if self.account: account = self.account else: - account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") + account_name = re.sub(r'Folder$', "", self.__name__) account = self.pyload.accountManager.getAccountPlugin(account_name) if account and hasattr(account, "PLUGIN_DOMAIN") and account.PLUGIN_DOMAIN: diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 80ef9a977..06715d2f1 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -158,7 +158,7 @@ class XFSHoster(SimpleHoster): action, inputs = self.parse_html_form('F1') if not inputs: - self.retry(msg=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) + self.retry(msg=self.info.get('error') or _("TEXTAREA F1 not found")) self.log_debug(inputs) @@ -184,7 +184,7 @@ class XFSHoster(SimpleHoster): header = self.load(m.group(1), just_header=True) if 'location' in header: #: Direct download link - self.link = header['location'] + self.link = header.get('location') def get_post_parameters(self): @@ -196,7 +196,7 @@ class XFSHoster(SimpleHoster): if not inputs: action, inputs = self.parse_html_form('F1') if not inputs: - self.retry(msg=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) + self.retry(msg=self.info.get('error') or _("TEXTAREA F1 not found")) self.log_debug(inputs) |