diff options
Diffstat (limited to 'module/plugins/internal')
-rw-r--r-- | module/plugins/internal/Account.py | 2 | ||||
-rw-r--r-- | module/plugins/internal/Addon.py | 14 | ||||
-rw-r--r-- | module/plugins/internal/Base.py | 44 | ||||
-rw-r--r-- | module/plugins/internal/Crypter.py | 23 | ||||
-rw-r--r-- | module/plugins/internal/Hoster.py | 50 | ||||
-rw-r--r-- | module/plugins/internal/MultiHoster.py | 4 | ||||
-rw-r--r-- | module/plugins/internal/Plugin.py | 89 | ||||
-rw-r--r-- | module/plugins/internal/SimpleCrypter.py | 12 | ||||
-rw-r--r-- | module/plugins/internal/SimpleHoster.py | 2 | ||||
-rw-r--r-- | module/plugins/internal/UnRar.py | 2 | ||||
-rw-r--r-- | module/plugins/internal/XFSCrypter.py | 6 | ||||
-rw-r--r-- | module/plugins/internal/XFSHoster.py | 6 |
12 files changed, 157 insertions, 97 deletions
diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index a0c46a6a1..a1ecdd76f 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -40,7 +40,7 @@ class Account(Plugin): #: Callback of periodical job task self.cb = None - self.interval = None + self.interval = self.PERIODICAL_INTERVAL self.init() diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 47c5584f5..bdde514c0 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -23,7 +23,7 @@ def threaded(fn): class Addon(Plugin): __name__ = "Addon" __type__ = "hook" #@TODO: Change to `addon` in 0.4.10 - __version__ = "0.11" + __version__ = "0.12" __status__ = "testing" __threaded__ = [] #@TODO: Remove in 0.4.10 @@ -53,7 +53,7 @@ class Addon(Plugin): #: Callback of periodical job task, used by HookManager self.cb = None - self.interval = None + self.interval = self.PERIODICAL_INTERVAL self.init() self.init_events() @@ -132,6 +132,14 @@ class Addon(Plugin): raise NotImplementedError + def save_info(self): + self.store("info", self.info) + + + def restore_info(self): + self.retrieve("info", self.info) + + @property def activated(self): """ @@ -154,6 +162,7 @@ class Addon(Plugin): #: Deprecated method, use `deactivate` instead (Remove in 0.4.10) def unload(self, *args, **kwargs): + self.save_info() return self.deactivate(*args, **kwargs) @@ -166,6 +175,7 @@ class Addon(Plugin): #: Deprecated method, use `activate` instead (Remove in 0.4.10) def coreReady(self, *args, **kwargs): + self.restore_info() return self.activate(*args, **kwargs) diff --git a/module/plugins/internal/Base.py b/module/plugins/internal/Base.py index 423e53695..e3aaf202b 100644 --- a/module/plugins/internal/Base.py +++ b/module/plugins/internal/Base.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -import re import inspect import mimetypes import os +import re import time import urlparse @@ -47,7 +47,7 @@ def check_abort(fn): class Base(Plugin): __name__ = "Base" __type__ = "base" - __version__ = "0.09" + __version__ = "0.11" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -64,6 +64,9 @@ class Base(Plugin): def __init__(self, pyfile): self._init(pyfile.m.core) + #: + self.premium = None + #: Engage wan reconnection self.wantReconnect = False #@TODO: Change to `want_reconnect` in 0.4.10 @@ -95,6 +98,9 @@ class Base(Plugin): #: Dict of the amount of retries already made self.retries = {} + self.init_base() + self.init() + def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) @@ -124,6 +130,10 @@ class Base(Plugin): return info + def init_base(self): + pass + + def init(self): """ Initialize the plugin (in addition to `__init__`) @@ -131,6 +141,10 @@ class Base(Plugin): pass + def setup_base(self): + pass + + def setup(self): """ Setup for enviroment and other things, called before downloading (possibly more than one time) @@ -156,16 +170,13 @@ class Base(Plugin): pass if self.account: - self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) - self.chunk_limit = -1 #: -1 for unlimited - self.resume_download = True - self.premium = self.account.info['data']['premium'] #: Don't use `self.account.premium` to avoid one unnecessary get_info call + self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) + self.premium = self.account.info['data']['premium'] #@NOTE: Avoid one unnecessary get_info call by `self.account.premium` here else: - self.req = self.pyload.requestFactory.getRequest(self.__name__) - self.chunk_limit = 1 - self.resume_download = False - self.premium = False + self.req = self.pyload.requestFactory.getRequest(self.__name__) + self.premium = False + self.setup_base() self.setup() @@ -306,7 +317,7 @@ class Base(Plugin): if msg: self.pyfile.error = msg else: - msg = self.pyfile.error or (self.info['error'] if 'error' in self.info else self.pyfile.getStatusName()) + msg = self.pyfile.error or self.info.get('error') or self.pyfile.getStatusName() raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 @@ -464,10 +475,11 @@ class Base(Plugin): value = value.strip() if key in header: - if type(header[key]) is list: - header[key].append(value) + header_key = header.get(key) + if type(header_key) is list: + header_key.append(value) else: - header[key] = [header[key], value] + header[key] = [header_key, value] else: header[key] = value @@ -475,7 +487,7 @@ class Base(Plugin): link = url elif header.get('location'): - location = self.fixurl(header['location'], url) + location = self.fixurl(header.get('location'), url) if header.get('code') == 302: link = location @@ -488,7 +500,7 @@ class Base(Plugin): extension = os.path.splitext(parse_name(url))[-1] if header.get('content-type'): - mimetype = header['content-type'].split(';')[0].strip() + mimetype = header.get('content-type').split(';')[0].strip() elif extension: mimetype = mimetypes.guess_type(extension, False)[0] or "application/octet-stream" diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index db13e445c..20322bb33 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.Base import Base, parse_name +from module.plugins.internal.Base import Base, check_abort, create_getInfo, parse_fileInfo +from module.plugins.internal.Plugin import parse_name from module.utils import save_path as safe_filename class Crypter(Base): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.12" + __version__ = "0.13" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -20,22 +21,12 @@ class Crypter(Base): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - def __init__(self, *args, **kwargs): - super(Crypter, self).__init__(*args, **kwargs) + def init_base(self): + self.packages = [] #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) + self.urls = [] #: List of urls, pyLoad will generate packagenames - #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) - self.packages = [] - - #: List of urls, pyLoad will generate packagenames - self.urls = [] - - self._setup() - self.init() - - - def _setup(self): - super(Crypter, self)._setup() + def setup_base(self): self.packages = [] self.urls = [] diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index c65946413..d81154554 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -5,7 +5,7 @@ from __future__ import with_statement import os import re -from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo, parse_fileInfo +from module.plugins.internal.Base import Base, check_abort, create_getInfo, parse_fileInfo from module.plugins.internal.Plugin import Fail, Retry, encode, exists, fixurl, parse_name from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename @@ -13,7 +13,7 @@ from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path a class Hoster(Base): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.36" + __version__ = "0.37" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -26,11 +26,15 @@ class Hoster(Base): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - def __init__(self, *args, **kwargs): - super(Hoster, self).__init__(*args, **kwargs) - + def init_base(self): #: Enable simultaneous processing of multiple downloads - self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 + self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 + + #: + self.chunk_limit = None + + #: + self.resume_download = False #: Location where the last call to download was saved self.last_download = None @@ -41,17 +45,19 @@ class Hoster(Base): #: Restart flag self.rst_free = False #@TODO: Recheck in 0.4.10 - self._setup() - self.init() - - - def _setup(self): - super(Hoster, self)._setup() + def setup_base(self): self.last_download = None self.last_check = None self.rst_free = False + if self.account: + self.chunk_limit = -1 #: -1 for unlimited + self.resume_download = True + else: + self.chunk_limit = 1 + self.resume_download = False + def load_account(self): if self.rst_free: @@ -76,7 +82,8 @@ class Hoster(Base): self.pyfile.setStatus("starting") try: - self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) #@TODO: Remove in 0.4.10 + self.log_debug("PROCESS URL " + self.pyfile.url, + "PLUGIN VERSION %s" % self.__version__) #@TODO: Remove in 0.4.10 self.process(self.pyfile) self.check_abort() @@ -94,7 +101,7 @@ class Hoster(Base): @check_abort - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True, resume=None, chunks=None): """ Downloads the content at url to download folder @@ -141,10 +148,16 @@ class Hoster(Base): self.check_abort() + chunks = min(self.pyload.config.get("download", "chunks"), chunks or self.chunk_limit or -1) + + if resume is None: + resume = self.resume_download + try: - newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, - chunks=self.get_chunk_count(), resume=self.resume_download, - progressNotify=self.pyfile.setProgress, disposition=disposition) + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, + cookies=cookies, chunks=chunks, resume=resume, + progressNotify=self.pyfile.setProgress, + disposition=disposition) finally: self.pyfile.size = self.req.size @@ -159,7 +172,8 @@ class Hoster(Base): os.rename(oldname_enc, newname_enc) except OSError, e: - self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e) + self.log_warning(_("Error renaming `%s` to `%s`") + % (newname, finalname), e) finalname = newname self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname)) diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index f5c3d091d..231615ae5 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -104,8 +104,8 @@ class MultiHoster(SimpleHoster): self.log_warning(_("Premium download failed")) self.restart(premium=False) - elif self.get_config("revertfailed", True) \ - and "new_module" in self.pyload.pluginManager.hosterPlugins[self.__name__]: + elif self.get_config("revertfailed", True) and \ + self.pyload.pluginManager.hosterPlugins[self.__name__].get('new_module'): hdict = self.pyload.pluginManager.hosterPlugins[self.__name__] tmp_module = hdict['new_module'] diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index d6eac7e58..a2392fdcc 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -6,6 +6,7 @@ import datetime import inspect import os import re +import sys import time import traceback import urllib @@ -17,17 +18,38 @@ if os.name is not "nt": import grp import pwd +from module.common.json_layer import json_dumps, json_loads from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 -from module.utils import fs_encode, fs_decode, html_unescape, parseFileSize as parse_size, save_join as fs_join +from module.utils import (fs_encode, fs_decode, get_console_encoding, html_unescape, + parseFileSize as parse_size, save_join as fs_join) #@TODO: Move to utils in 0.4.10 -def decode(string, encoding='utf8'): - """ Decode string to unicode with utf8 """ +def isiterable(obj): + return hasattr(obj, "__iter__") + + +#@TODO: Move to utils in 0.4.10 +def decode(string, encoding=None): + """Encoded string (default to UTF-8) -> unicode string""" if type(string) is str: - return string.decode(encoding, "replace") + try: + res = unicode(string, encoding or "utf-8") + + except UnicodeDecodeError, e: + if encoding: + raise UnicodeDecodeError(e) + + encoding = get_console_encoding(sys.stdout.encoding) + res = unicode(string, encoding) + + elif type(string) is unicode: + res = string + else: - return unicode(string) + res = unicode(string) + + return res #@TODO: Remove in 0.4.10 @@ -36,12 +58,18 @@ def _decode(*args, **kwargs): #@TODO: Move to utils in 0.4.10 -def encode(string, encoding='utf8'): - """ Decode string to utf8 """ +def encode(string, encoding=None, decoding=None): + """Unicode or decoded string -> encoded string (default to UTF-8)""" if type(string) is unicode: - return string.encode(encoding, "replace") + res = string.encode(encoding or "utf-8") + + elif type(string) is str: + res = encode(decode(string, decoding), encoding) + else: - return str(string) + res = str(string) + + return res #@TODO: Move to utils in 0.4.10 @@ -186,11 +214,11 @@ def parse_html_tag_attr_value(attr_name, tag): def parse_html_form(attr_str, html, input_names={}): for form in re.finditer(r"(?P<TAG><form[^>]*%s[^>]*>)(?P<CONTENT>.*?)</?(form|body|html)[^>]*>" % attr_str, - html, re.S | re.I): + html, re.I | re.S): inputs = {} action = parse_html_tag_attr_value("action", form.group('TAG')) - for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.S | re.I): + for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.I | re.S): name = parse_html_tag_attr_value("name", inputtag.group(1)) if name: value = parse_html_tag_attr_value("value", inputtag.group(1)) @@ -234,7 +262,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.52" + __version__ = "0.53" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -301,7 +329,7 @@ class Plugin(object): def log_error(self, *args, **kwargs): self._log("error", self.__type__, self.__name__, args) - if kwargs.get('trace'): + if self.pyload.debug and kwargs.get('trace', True): print "Traceback (most recent call last):" traceback.print_stack() @@ -338,12 +366,6 @@ class Plugin(object): self.log_warning(_("Setting owner and group failed"), e) - def get_chunk_count(self): - if self.chunk_limit <= 0: - return self.pyload.config.get("download", "chunks") - return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) - - def set_config(self, option, value, plugin=None): """ Set config value for current plugin @@ -374,14 +396,26 @@ class Plugin(object): """ Saves a value persistently to the database """ - self.pyload.db.setStorage(self.__name__, key, value) + value = map(decode, value) if isiterable(value) else decode(value) + entry = json_dumps(value).encode('base64') + self.pyload.db.setStorage(self.__name__, key, entry) - def retrieve(self, key, default=None): + def retrieve(self, key=None, default=None): """ Retrieves saved value or dict of all saved entries if key is None """ - return self.pyload.db.getStorage(self.__name__, key) or default + entry = self.pyload.db.getStorage(self.__name__, key) + + if entry: + if type(entry) is dict: + value = dict((k, json_loads(v.decode('base64'))) for k, v in value.items()) or default + else: + value = json_loads(entry.decode('base64')) or default + else: + value = entry + + return value def delete(self, key): @@ -456,13 +490,13 @@ class Plugin(object): frame = inspect.currentframe() try: - framefile = fs_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + framefile = fs_join("tmp", self.__name__, "%s_line%s.dump.html" % + (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) if not exists(os.path.join("tmp", self.__name__)): os.makedirs(os.path.join("tmp", self.__name__)) with open(framefile, "wb") as f: - f.write(encode(html)) except IOError, e: @@ -487,10 +521,11 @@ class Plugin(object): value = value.strip() if key in header: - if type(header[key]) is list: - header[key].append(value) + header_key = header.get(key) + if type(header_key) is list: + header_key.append(value) else: - header[key] = [header[key], value] + header[key] = [header_key, value] else: header[key] = value diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index d4ecd3117..ba4235072 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.66" + __version__ = "0.67" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -82,20 +82,16 @@ class SimpleCrypter(Crypter, SimpleHoster): #@TODO: Remove in 0.4.10 def _setup(self): orig_name = self.__name__ - self.__name__ = (orig_name + ".py").replace("Folder.py", "").replace(".py", "") - + self.__name__ = re.sub(r'Folder$', "", self.__name__) super(SimpleCrypter, self)._setup() - self.__name__ = orig_name #@TODO: Remove in 0.4.10 def load_account(self): orig_name = self.__name__ - self.__name__ = (orig_name + ".py").replace("Folder.py", "").replace(".py", "") - + self.__name__ = re.sub(r'Folder$', "", self.__name__) super(SimpleCrypter, self).load_account() - self.__name__ = orig_name @@ -106,7 +102,7 @@ class SimpleCrypter(Crypter, SimpleHoster): header = self.load(redirect, just_header=True) if header.get('location'): - self.link = header['location'] + self.link = header.get('location') else: break else: diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 0f27ecc91..cf74c6646 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -270,7 +270,7 @@ class SimpleHoster(Hoster): if not self.link and not self.last_download: self.preload() - if 'status' not in self.info or self.info['status'] is 3: #@TODO: Recheck in 0.4.10 + if self.info.get('status', 3) is 3: #@TODO: Recheck in 0.4.10 self.check_info() if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index f31a386a7..6f85c286a 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -38,7 +38,7 @@ class UnRar(Extractor): re_multipart = re.compile(r'\.(part|r)(\d+)(?:\.rar)?(\.rev|\.bad)?', re.I) re_filefixed = re.compile(r'Building (.+)') - re_filelist = re.compile(r'^(.)(\s*[\w\-.]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.M|re.I) + re_filelist = re.compile(r'^(.)(\s*[\w\-.]+)\s+(\d+\s+)+(?:\d+\%\s+)?[\d\-]{8}\s+[\d\:]{5}', re.I | re.M) re_wrongpwd = re.compile(r'password', re.I) re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error|corrupt', re.I) diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index dd2fba568..7ec9f3895 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +import re + from module.plugins.internal.Plugin import set_cookie from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo @@ -7,7 +9,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.17" + __version__ = "0.18" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -49,7 +51,7 @@ class XFSCrypter(SimpleCrypter): if self.account: account = self.account else: - account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") + account_name = re.sub(r'Folder$', "", self.__name__) account = self.pyload.accountManager.getAccountPlugin(account_name) if account and hasattr(account, "PLUGIN_DOMAIN") and account.PLUGIN_DOMAIN: diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 80ef9a977..06715d2f1 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -158,7 +158,7 @@ class XFSHoster(SimpleHoster): action, inputs = self.parse_html_form('F1') if not inputs: - self.retry(msg=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) + self.retry(msg=self.info.get('error') or _("TEXTAREA F1 not found")) self.log_debug(inputs) @@ -184,7 +184,7 @@ class XFSHoster(SimpleHoster): header = self.load(m.group(1), just_header=True) if 'location' in header: #: Direct download link - self.link = header['location'] + self.link = header.get('location') def get_post_parameters(self): @@ -196,7 +196,7 @@ class XFSHoster(SimpleHoster): if not inputs: action, inputs = self.parse_html_form('F1') if not inputs: - self.retry(msg=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) + self.retry(msg=self.info.get('error') or _("TEXTAREA F1 not found")) self.log_debug(inputs) |