diff options
author | Jochen Oberreiter <joberreiter@users.noreply.github.com> | 2015-09-26 18:24:29 +0200 |
---|---|---|
committer | Jochen Oberreiter <joberreiter@users.noreply.github.com> | 2015-09-26 18:24:29 +0200 |
commit | 93a0c1d930520c055eae766b5dad305111a02c4d (patch) | |
tree | 8ac3f9e6dcf682775f5b170f2a9ca40eb7c0f8fc /module/plugins/internal | |
parent | Spare plugin updates (diff) | |
parent | Merge pull request #1850 from chaosblog/patch-2 (diff) | |
download | pyload-93a0c1d930520c055eae766b5dad305111a02c4d.tar.xz |
Merge pull request #1 from pyload/stable
Merge actual version
Diffstat (limited to 'module/plugins/internal')
-rw-r--r-- | module/plugins/internal/Account.py | 12 | ||||
-rw-r--r-- | module/plugins/internal/Addon.py | 23 | ||||
-rw-r--r-- | module/plugins/internal/Captcha.py | 21 | ||||
-rw-r--r-- | module/plugins/internal/Container.py | 20 | ||||
-rw-r--r-- | module/plugins/internal/Crypter.py | 15 | ||||
-rw-r--r-- | module/plugins/internal/Extractor.py | 56 | ||||
-rw-r--r-- | module/plugins/internal/Hoster.py | 310 | ||||
-rw-r--r-- | module/plugins/internal/OCR.py | 6 | ||||
-rw-r--r-- | module/plugins/internal/Plugin.py | 116 | ||||
-rw-r--r-- | module/plugins/internal/SevenZip.py | 32 | ||||
-rw-r--r-- | module/plugins/internal/SimpleCrypter.py | 13 | ||||
-rw-r--r-- | module/plugins/internal/SimpleHoster.py | 77 | ||||
-rw-r--r-- | module/plugins/internal/UnRar.py | 43 | ||||
-rw-r--r-- | module/plugins/internal/UnZip.py | 14 | ||||
-rw-r--r-- | module/plugins/internal/XFSAccount.py | 21 | ||||
-rw-r--r-- | module/plugins/internal/XFSCrypter.py | 4 | ||||
-rw-r--r-- | module/plugins/internal/XFSHoster.py | 14 |
17 files changed, 415 insertions, 382 deletions
diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 2713e8da4..de338cd33 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -13,7 +13,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.17" + __version__ = "0.18" __status__ = "testing" __description__ = """Base account plugin""" @@ -190,7 +190,7 @@ class Account(Plugin): def get_info(self, user, reload=False): """ Retrieve account infos for an user, do **not** overwrite this method!\\ - just use it to retrieve infos in hoster plugins. see `parse_info` + just use it to retrieve infos in hoster plugins. see `grab_info` :param user: username :param reload: reloads cached account information @@ -235,7 +235,7 @@ class Account(Plugin): try: self.req = self.get_request(user) - extra_info = self.parse_info(user, info['login']['password'], info, self.req) + extra_info = self.grab_info(user, info['login']['password'], info, self.req) if extra_info and isinstance(extra_info, dict): info['data'].update(extra_info) @@ -253,7 +253,7 @@ class Account(Plugin): return info - def parse_info(self, user, password, info, req): + def grab_info(self, user, password, info, req): """ This should be overwritten in account plugin and retrieving account information for user @@ -270,8 +270,8 @@ class Account(Plugin): return [self.getAccountData(user, *args, **kwargs) for user, info in self.info.items()] - def login_fail(self, reason=_("Login handshake has failed")): - return self.fail(reason) + def fail_login(self, msg=_("Login handshake has failed")): + return self.fail(msg) def get_request(self, user=None): diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 45ca98eac..5150e88f6 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -import traceback - from module.plugins.internal.Plugin import Plugin @@ -25,7 +23,7 @@ def threaded(fn): class Addon(Plugin): __name__ = "Addon" __type__ = "hook" #@TODO: Change to `addon` in 0.4.10 - __version__ = "0.04" + __version__ = "0.06" __status__ = "testing" __config__ = [] #: [("name", "type", "desc", "default")] @@ -57,6 +55,12 @@ class Addon(Plugin): self.init_events() + #@TODO: Remove in 0.4.10 + def _log(self, level, plugintype, pluginname, messages): + plugintype = "addon" if plugintype is "hook" else plugintype + return super(Addon, self)._log(level, plugintype, pluginname, messages) + + def init_events(self): if self.event_map: for event, funcs in self.event_map.items(): @@ -97,8 +101,6 @@ class Addon(Plugin): except Exception, e: self.log_error(_("Error executing periodical task: %s") % e) - if self.pyload.debug: - traceback.print_exc() self.cb = self.pyload.scheduler.addJob(self.interval, self._periodical, [threaded], threaded=threaded) @@ -107,20 +109,17 @@ class Addon(Plugin): pass - def __repr__(self): - return "<Addon %s>" % self.__name__ - - - def is_activated(self): + @property + def activated(self): """ Checks if addon is activated """ return self.get_config("activated") - #: Deprecated method, use `is_activated` instead (Remove in 0.4.10) + #: Deprecated method, use `activated` property instead (Remove in 0.4.10) def isActivated(self, *args, **kwargs): - return self.is_activated(*args, **kwargs) + return self.activated def deactivate(self): diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index c08050ee8..d2be21a58 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -4,7 +4,6 @@ from __future__ import with_statement import os import time -import traceback from module.plugins.internal.Plugin import Plugin @@ -12,7 +11,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.42" + __version__ = "0.44" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -50,18 +49,18 @@ class Captcha(Plugin): pass - def decrypt(self, url, get={}, post={}, ref=False, cookies=False, decode=False, + def decrypt(self, url, get={}, post={}, ref=False, cookies=True, decode=False, input_type='jpg', output_type='textual', ocr=True, timeout=120): img = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode) return self._decrypt(img, input_type, output_type, ocr, timeout) #@TODO: Definitely choose a better name for this method! - def _decrypt(self, raw, input_type='jpg', output_type='textual', ocr=False, timeout=120): + def _decrypt(self, data, input_type='jpg', output_type='textual', ocr=False, timeout=120): """ Loads a captcha and decrypts it with ocr, plugin, user input - :param raw: image raw data + :param data: image raw data :param get: get part for request :param post: post part for request :param cookies: True if cookies should be enabled @@ -77,7 +76,7 @@ class Captcha(Plugin): time_ref = ("%.2f" % time.time())[-6:].replace(".", "") with open(os.path.join("tmp", "captcha_image_%s_%s.%s" % (self.plugin.__name__, time_ref, input_type)), "wb") as tmp_img: - tmp_img.write(raw) + tmp_img.write(data) if ocr: if isinstance(ocr, basestring): @@ -90,14 +89,13 @@ class Captcha(Plugin): captchaManager = self.pyload.captchaManager try: - self.task = captchaManager.newTask(raw, input_type, tmp_img.name, output_type) + self.task = captchaManager.newTask(data, input_type, tmp_img.name, output_type) captchaManager.handleCaptcha(self.task) self.task.setWaiting(max(timeout, 50)) #@TODO: Move to `CaptchaManager` in 0.4.10 while self.task.isWaiting(): - if self.plugin.pyfile.abort: - self.plugin.abort() + self.plugin.check_abort() time.sleep(1) finally: @@ -108,7 +106,7 @@ class Captcha(Plugin): elif not self.task.result: self.invalid() - self.plugin.retry(reason=_("No captcha result obtained in appropiate time")) + self.plugin.retry(msg=_("No captcha result obtained in appropiate time")) result = self.task.result @@ -118,9 +116,8 @@ class Captcha(Plugin): except OSError, e: self.log_warning(_("Error removing: %s") % tmp_img.name, e) - traceback.print_exc() - self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here? + #self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here? return result diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 729592a0d..430590421 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -4,7 +4,6 @@ from __future__ import with_statement import os import re -import traceback from module.plugins.internal.Crypter import Crypter from module.plugins.internal.Plugin import exists @@ -14,7 +13,7 @@ from module.utils import save_join as fs_join class Container(Crypter): __name__ = "Container" __type__ = "container" - __version__ = "0.06" + __version__ = "0.07" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -44,11 +43,6 @@ class Container(Crypter): self._create_packages() - #: Deprecated method, use `_load2disk` instead (Remove in 0.4.10) - def loadToDisk(self, *args, **kwargs): - return self._load2disk(*args, **kwargs) - - def _load2disk(self): """ Loads container to disk if its stored remotely and overwrite url, @@ -63,20 +57,18 @@ class Container(Crypter): f.write(content) except IOError, e: - self.fail(str(e)) #@TODO: Remove `str` in 0.4.10 + self.fail(e) else: self.pyfile.name = os.path.basename(self.pyfile.url) + if not exists(self.pyfile.url): if exists(fs_join(pypath, self.pyfile.url)): self.pyfile.url = fs_join(pypath, self.pyfile.url) else: self.fail(_("File not exists")) - - - #: Deprecated method, use `delete_tmp` instead (Remove in 0.4.10) - def deleteTmp(self, *args, **kwargs): - return self.delete_tmp(*args, **kwargs) + else: + self.data = self.pyfile.url def delete_tmp(self): @@ -87,5 +79,3 @@ class Container(Crypter): os.remove(self.pyfile.url) except OSError, e: self.log_warning(_("Error removing: %s") % self.pyfile.url, e) - if self.pyload.debug: - traceback.print_exc() diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index d0e8eb1b4..2033b67df 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -1,15 +1,13 @@ # -*- coding: utf-8 -*- -import urlparse - -from module.plugins.internal.Hoster import Hoster, _fixurl +from module.plugins.internal.Hoster import Hoster, parse_name from module.utils import save_path as safe_filename class Crypter(Hoster): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.07" + __version__ = "0.08" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -78,13 +76,14 @@ class Crypter(Hoster): "%d links" % len(links), "Saved to folder: %s" % folder if folder else "Saved to download folder") - pid = self.pyload.api.addPackage(name, map(self.fixurl, links), package_queue) + links = map(self.fixurl, links) + pid = self.pyload.api.addPackage(name, links, package_queue) if package_password: self.pyload.api.setPackageData(pid, {'password': package_password}) #: Workaround to do not break API addPackage method - set_folder = lambda x: self.pyload.api.setPackageData(pid, {'folder': x or ""}) + set_folder = lambda x: self.pyload.api.setPackageData(pid, {'folder': safe_filename(x) or ""}) if use_subfolder: if not subfolder_per_package: @@ -93,9 +92,9 @@ class Crypter(Hoster): elif not folder_per_package or name is not folder: if not folder: - folder = urlparse.urlparse(_fixurl(name)).path.split("/")[-1] + folder = parse_name(name) - set_folder(safe_filename(folder)) + set_folder(folder) self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder}) elif folder_per_package: diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 7f5212090..f21fe473c 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -22,7 +22,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.33" + __version__ = "0.34" __status__ = "testing" __description__ = """Base extractor plugin""" @@ -43,15 +43,9 @@ class Extractor(Plugin): @classmethod - def is_multipart(cls, filename): - return False - - - @classmethod def find(cls): """ Check if system statisfy dependencies - :return: boolean """ pass @@ -72,9 +66,15 @@ class Extractor(Plugin): if pname not in processed: processed.append(pname) targets.append((fname, id, fout)) + return targets + @property + def target(self): + return fs_encode(self.filename) + + def __init__(self, plugin, filename, out, fullpath=True, overwrite=False, @@ -119,53 +119,29 @@ class Extractor(Plugin): (self.__name__,) + messages) - def check(self): + def verify(self, password=None): """ - Quick Check by listing content of archive. - Raises error if password is needed, integrity is questionable or else. - - :raises PasswordError - :raises CRCError - :raises ArchiveError + Testing with Extractors built-in method + Raise error if password is needed, integrity is questionable or else """ - raise NotImplementedError - - - def verify(self): - """ - Testing with Extractors buildt-in method - Raises error if password is needed, integrity is questionable or else. - - :raises PasswordError - :raises CRCError - :raises ArchiveError - """ - raise NotImplementedError + pass def repair(self): - return None + return False def extract(self, password=None): """ - Extract the archive. Raise specific errors in case of failure. - - :param progress: Progress function, call this to update status - :param password password to use - :raises PasswordError - :raises CRCError - :raises ArchiveError - :return: + Extract the archive + Raise specific errors in case of failure """ raise NotImplementedError - def get_delete_files(self): + def items(self): """ - Return list of files to delete, do *not* delete them here. - - :return: List with paths of files to delete + Return list of archive parts """ return [self.filename] diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index b397a92a6..5d0a64f1a 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -7,21 +7,21 @@ import mimetypes import os import random import time -import traceback import urlparse from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, encode, exists, fixurl as _fixurl, replace_patterns, - seconds_to_midnight, set_cookie, set_cookies, parse_html_form, - parse_html_tag_attr_value, timestamp) + chunks, decode, encode, exists, parse_html_form, + parse_html_tag_attr_value, parse_name, + replace_patterns, seconds_to_midnight, + set_cookie, set_cookies, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename #@TODO: Remove in 0.4.10 def parse_fileInfo(klass, url="", html=""): info = klass.get_info(url, html) - return info['name'], info['size'], info['status'], info['url'] + return encode(info['name']), info['size'], info['status'], info['url'] #@TODO: Remove in 0.4.10 @@ -44,7 +44,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.20" + __version__ = "0.28" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -74,7 +74,6 @@ class Hoster(Plugin): #: Account handler instance, see :py:class:`Account` self.account = None - self.user = None self.req = None #: Browser instance, see `network.Browser` #: Associated pyfile instance, see `PyFile` @@ -105,15 +104,21 @@ class Hoster(Plugin): self.init() + def _log(self, level, plugintype, pluginname, messages): + log = getattr(self.pyload.log, level) + msg = u" | ".join(decode(a).strip() for a in messages if a) + log("%(plugintype)s %(pluginname)s[%(id)s]: %(msg)s" + % {'plugintype': plugintype.upper(), + 'pluginname': pluginname, + 'id' : self.pyfile.id, + 'msg' : msg}) + + @classmethod def get_info(cls, url="", html=""): - url = _fixurl(url) - url_p = urlparse.urlparse(url) - return {'name' : (url_p.path.split('/')[-1] or - url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or - url_p.netloc.split('.', 1)[0]), + return {'name' : parse_name(url), 'size' : 0, - 'status': 3 if url else 8, + 'status': 3 if url.strip() else 8, 'url' : url} @@ -132,34 +137,26 @@ class Hoster(Plugin): def _setup(self): + #@TODO: Remove in 0.4.10 + self.html = "" + self.last_download = "" + self.pyfile.error = "" + if self.account: - self.req = self.pyload.requestFactory.getRequest(self.__name__, self.user) + self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) self.chunk_limit = -1 #: -1 for unlimited self.resume_download = True - self.premium = self.account.is_premium(self.user) + self.premium = self.account.premium else: self.req = self.pyload.requestFactory.getRequest(self.__name__) self.chunk_limit = 1 self.resume_download = False self.premium = False + return self.setup() - def load_account(self): - if self.req: - self.req.close() - - if not self.account: - self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) - if self.account: - if not self.user: - self.user = self.account.select()[0] - - if not self.user or not self.account.is_logged(self.user, True): - self.account = False - - - def preprocessing(self, thread): + def _process(self, thread): """ Handles important things to do before starting """ @@ -172,19 +169,36 @@ class Hoster(Plugin): self.retry_free = False self._setup() - self.setup() + self.pyfile.setStatus("starting") self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 - if self.pyfile.abort: - self.abort() + self.check_abort() - self.pyfile.setStatus("starting") self.log_debug("PROCESS URL " + self.pyfile.url, "PLUGIN VERSION %s" % self.__version__) - return self.process(self.pyfile) + #: Deprecated method, use `_process` instead (Remove in 0.4.10) + def preprocessing(self, *args, **kwargs): + return self._process(*args, **kwargs) + + + def load_account(self): + if self.req: + self.req.close() + + if not self.account: + self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) + + if self.account: + if not hasattr(self.account, 'user'): #@TODO: Move to `Account` in 0.4.10 + self.account.user = self.account.select()[0] + + if not hasattr(self.account, 'logged'): + self.account = False + + def process(self, pyfile): """ The 'main' method of every plugin, you **have to** overwrite it @@ -193,12 +207,13 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): - reconnect = bool(reconnect) + if reconnect: + self.log_info(_("Requesting line reconnection...")) + else: + self.log_debug("Reconnect: %s" % reconnect) - self.log_info(_("RECONNECT ") + ("enabled" if reconnect else "disabled")) self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) - - self.wantReconnect = reconnect + self.wantReconnect = bool(reconnect) def set_wait(self, seconds, reconnect=None): @@ -211,7 +226,7 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_info(_("WAIT %d seconds") % wait_time) + self.log_info(_("Waiting %d seconds...") % wait_time) self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until @@ -242,15 +257,12 @@ class Hoster(Plugin): self.log_warning("Ignore reconnection due logged account") while pyfile.waitUntil > time.time(): - if pyfile.abort: - self.abort() - + self.check_abort() time.sleep(2) else: while pyfile.waitUntil > time.time(): - if pyfile.abort: - self.abort() + self.check_abort() if self.thread.m.reconnecting.isSet(): self.waiting = False @@ -264,91 +276,113 @@ class Hoster(Plugin): pyfile.status = status #@NOTE: Remove in 0.4.10 - def skip(self, reason=""): + def skip(self, msg=""): """ - Skip and give reason + Skip and give msg """ - raise Skip(encode(reason)) #@TODO: Remove `encode` in 0.4.10 + raise Skip(encode(msg or self.pyfile.error)) #@TODO: Remove `encode` in 0.4.10 - def abort(self, reason=""): + #@TODO: Remove in 0.4.10 + def fail(self, msg): """ - Abort and give reason + Fail and give msg """ - #@TODO: Remove in 0.4.10 - if reason: - self.pyfile.error = encode(reason) + msg = msg.strip() + + if msg: + self.pyfile.error = msg + else: + msg = self.pyfile.error + + raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 + + + def error(self, msg="", type=_("Parse")): + type = _("%s error") % type.strip().capitalize() if type else _("Unknown") + msg = _("%(type)s: %(msg)s | Plugin may be out of date" + % {'type': type, 'msg': msg or self.pyfile.error}) + + self.fail(msg) + + + def abort(self, msg=""): + """ + Abort and give msg + """ + if msg: #@TODO: Remove in 0.4.10 + self.pyfile.error = encode(msg) raise Abort - def offline(self, reason=""): + #@TODO: Recheck in 0.4.10 + def offline(self, msg=""): """ Fail and indicate file is offline """ - #@TODO: Remove in 0.4.10 - if reason: - self.pyfile.error = encode(reason) - - raise Fail("offline") + self.fail("offline") - def temp_offline(self, reason=""): + #@TODO: Recheck in 0.4.10 + def temp_offline(self, msg=""): """ Fail and indicates file ist temporary offline, the core may take consequences """ - #@TODO: Remove in 0.4.10 - if reason: - self.pyfile.error = encode(reason) + self.fail("temp. offline") - raise Fail("temp. offline") - - def retry(self, max_tries=5, wait_time=1, reason=""): + def retry(self, attemps=5, delay=1, msg=""): """ Retries and begin again from the beginning - :param max_tries: number of maximum retries - :param wait_time: time to wait in seconds - :param reason: reason for retrying, will be passed to fail if max_tries reached + :param attemps: number of maximum retries + :param delay: time to wait in seconds + :param msg: msg for retrying, will be passed to fail if attemps value was reached """ id = inspect.currentframe().f_back.f_lineno if id not in self.retries: self.retries[id] = 0 - if 0 < max_tries <= self.retries[id]: - self.fail(reason or _("Max retries reached")) + if 0 < attemps <= self.retries[id]: + self.fail(msg or _("Max retries reached")) - self.wait(wait_time, False) + self.wait(delay, False) self.retries[id] += 1 - raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 + raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def restart(self, reason=None, nopremium=False): - if not reason: - reason = _("Fallback to free download") if nopremium else _("Restart") + def restart(self, msg=None, nopremium=False): + if not msg: + msg = _("Fallback to free download") if nopremium else _("Restart") if nopremium: if self.premium: self.retry_free = True else: - self.fail("%s | %s" % (reason, _("Download was already free"))) + self.fail("%s | %s" % (msg, _("Download was already free"))) - raise Retry(encode(reason)) #@TODO: Remove `encode` in 0.4.10 + raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def fixurl(self, url): - url = _fixurl(url) + def fixurl(self, url, baseurl=None): + if not baseurl: + baseurl = self.pyfile.url if not urlparse.urlparse(url).scheme: - url_p = urlparse.urlparse(self.pyfile.url) + url_p = urlparse.urlparse(baseurl) baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) url = urlparse.urljoin(baseurl, url) return url + def load(self, *args, **kwargs): + self.check_abort() + return super(Hoster, self).load(*args, **kwargs) + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): """ Downloads the content at url to download folder @@ -362,20 +396,13 @@ class Hoster(Plugin): the filename will be changed if needed :return: The location where the file was saved """ - if self.pyfile.abort: - self.abort() - - url = self.fixurl(url) - - if not url or not isinstance(url, basestring): - self.fail(_("No url given")) + self.check_abort() if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) - name = _fixurl(self.pyfile.name) - self.pyfile.name = urlparse.urlparse(name).path.split('/')[-1] or name + self.pyfile.name = parse_name(self.pyfile.name) #: Safe check self.captcha.correct() self.check_for_same_files() @@ -388,6 +415,7 @@ class Hoster(Plugin): if not exists(download_location): try: os.makedirs(download_location) + except Exception, e: self.fail(e) @@ -398,8 +426,7 @@ class Hoster(Plugin): self.pyload.hookManager.dispatchEvent("download_start", self.pyfile, url, filename) - if self.pyfile.abort: - self.abort() + self.check_abort() try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, @@ -410,9 +437,9 @@ class Hoster(Plugin): #@TODO: Recheck in 0.4.10 if disposition and newname: - finalname = urlparse.urlparse(newname).path.split('/')[-1].split(' filename*=')[0] + finalname = parse_name(newname).split(' filename*=')[0] - if finalname != newname != self.pyfile.name: + if finalname != newname: try: os.rename(fs_join(location, newname), fs_join(location, finalname)) @@ -421,8 +448,9 @@ class Hoster(Plugin): finalname = newname self.log_info(_("`%s` saved as `%s`") % (self.pyfile.name, finalname)) - self.pyfile.name = finalname - filename = os.path.join(location, finalname) + + self.pyfile.name = finalname + filename = os.path.join(location, finalname) self.set_permissions(fs_encode(filename)) @@ -431,9 +459,55 @@ class Hoster(Plugin): return self.last_download - def check_download(self, rules, delete=False, file_size=0, size_tolerance=1024, read_size=1048576): + def check_abort(self): + if not self.pyfile.abort: + return + + if self.pyfile.hasStatus("failed"): + self.fail() + + elif self.pyfile.hasStatus("skipped"): + self.skip(self.pyfile.statusname) + + elif self.pyfile.hasStatus("offline"): + self.offline() + + elif self.pyfile.hasStatus("temp. offline"): + self.temp_offline() + + else: + self.abort() + + + def check_filesize(self, file_size, size_tolerance=1024): + """ + Checks the file size of the last downloaded file + + :param file_size: expected file size + :param size_tolerance: size check tolerance + """ + if not self.last_download: + return + + download_size = os.stat(fs_encode(self.last_download)).st_size + + if download_size < 1: + self.fail(_("Empty file")) + + elif file_size > 0: + diff = abs(file_size - download_size) + + if diff > size_tolerance: + self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s") + % (file_size, download_size)) + + elif diff != 0: + self.log_warning(_("File size is not equal to expected size")) + + + def check_download(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): """ - Checks the content of the last downloaded file, re match is saved to `lastCheck` + Checks the content of the last downloaded file, re match is saved to `last_check` :param rules: dict with names and rules to match (compiled regexp or strings) :param delete: delete if matched @@ -446,26 +520,10 @@ class Hoster(Plugin): last_download = fs_encode(self.last_download) if not self.last_download or not exists(last_download): - self.last_download = "" self.fail(self.pyfile.error or _("No file downloaded")) try: - download_size = os.stat(last_download).st_size - - if download_size < 1: - do_delete = True - self.fail(_("Empty file")) - - elif file_size > 0: - diff = abs(file_size - download_size) - - if diff > size_tolerance: - do_delete = True - self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s") - % (file_size, download_size)) - - elif diff != 0: - self.log_warning(_("File size is not equal to expected size")) + self.check_filesize(file_size, size_tolerance) with open(last_download, "rb") as f: content = f.read(read_size) @@ -491,12 +549,10 @@ class Hoster(Plugin): except OSError, e: self.log_warning(_("Error removing: %s") % last_download, e) - if self.pyload.debug: - traceback.print_exc() else: + self.log_info(_("File deleted: ") + self.last_download) self.last_download = "" - self.log_info(_("File deleted")) def direct_link(self, url, follow_location=None): @@ -519,7 +575,7 @@ class Hoster(Plugin): except Exception: #: Bad bad bad... rewrite this part in 0.4.10 res = self.load(url, just_header=True, - req=self.pyload.requestFactory.getRequest()) + req=self.pyload.requestFactory.getRequest(self.__name__)) header = {'code': req.code} for line in res.splitlines(): @@ -543,12 +599,7 @@ class Hoster(Plugin): link = url elif 'location' in header and header['location']: - location = header['location'] - - if not urlparse.urlparse(location).scheme: - url_p = urlparse.urlparse(url) - baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) - location = urlparse.urljoin(baseurl, location) + location = self.fixurl(header['location'], url) if 'code' in header and header['code'] == 302: link = location @@ -558,7 +609,7 @@ class Hoster(Plugin): continue else: - extension = os.path.splitext(urlparse.urlparse(url).path.split('/')[-1])[-1] + extension = os.path.splitext(parse_name(url))[-1] if 'content-type' in header and header['content-type']: mimetype = header['content-type'].split(';')[0].strip() @@ -579,6 +630,7 @@ class Hoster(Plugin): else: try: self.log_error(_("Too many redirects")) + except Exception: pass @@ -593,15 +645,17 @@ class Hoster(Plugin): if not self.account: return True - traffic = self.account.get_data(self.user, True)['trafficleft'] + traffic = self.account.get_data(self.account.user, True)['trafficleft'] if traffic is None: return False + elif traffic == -1: return True + else: size = self.pyfile.size / 1024 - self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.user, traffic)) + self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic)) return size <= traffic diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index b24b3058b..3e5afae69 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -12,7 +12,6 @@ import logging import os import subprocess # import tempfile -import traceback from module.plugins.internal.Plugin import Plugin from module.utils import save_join as fs_join @@ -128,6 +127,7 @@ class OCR(Plugin): try: with open(tmpTxt.name, 'r') as f: self.result_captcha = f.read().replace("\n", "") + except Exception: self.result_captcha = "" @@ -137,10 +137,9 @@ class OCR(Plugin): os.remove(tmpTxt.name) if subset and (digits or lowercase or uppercase): os.remove(tmpSub.name) + except OSError, e: self.log_warning(e) - if self.pyload.debug: - traceback.print_exc() def recognize(self, name): @@ -194,6 +193,7 @@ class OCR(Plugin): count += 1 if pixels[x, y - 1] != 255: count += 1 + except Exception: pass diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 7b45c40a8..51192d8c9 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -6,7 +6,10 @@ import datetime import inspect import os import re +import sys +import traceback import urllib +import urlparse if os.name != "nt": import grp @@ -22,7 +25,7 @@ def decode(string, encoding='utf8'): if type(string) is str: return string.decode(encoding, "replace") else: - return string + return unicode(string) #@TODO: Move to utils in 0.4.10 @@ -31,7 +34,7 @@ def encode(string, encoding='utf8'): if type(string) is unicode: return string.encode(encoding, "replace") else: - return string + return str(string) #@TODO: Move to utils in 0.4.10 @@ -47,8 +50,19 @@ def exists(path): #@TODO: Move to utils in 0.4.10 -def fixurl(url): - return html_unescape(urllib.unquote(url.decode('unicode-escape'))).strip().rstrip('/') +def parse_name(url): + url = urllib.unquote(url) + url = url.decode('unicode-escape') + url = html_unescape(url) + url = urllib.quote(url) + + url_p = urlparse.urlparse(url.strip().rstrip('/')) + + name = (url_p.path.split('/')[-1] or + url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or + url_p.netloc.split('.', 1)[0]) + + return urllib.unquote(name) #@TODO: Move to utils in 0.4.10 @@ -56,22 +70,35 @@ def timestamp(): return int(time.time() * 1000) -def seconds_to_midnight(gmt=0): - now = datetime.datetime.utcnow() + datetime.timedelta(hours=gmt) - - if now.hour == 0 and now.minute < 10: - midnight = now +#@TODO: Move to utils in 0.4.10 +def which(program): + """ + Works exactly like the unix command which + Courtesy of http://stackoverflow.com/a/377028/675646 + """ + isExe = lambda x: os.path.isfile(x) and os.access(x, os.X_OK) + + fpath, fname = os.path.split(program) + + if fpath: + if isExe(program): + return program else: - midnight = now + datetime.timedelta(days=1) + for path in os.environ['PATH'].split(os.pathsep): + exe_file = os.path.join(path.strip('"'), program) + if isExe(exe_file): + return exe_file - td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now - if hasattr(td, 'total_seconds'): - res = td.total_seconds() - else: #@NOTE: work-around for python 2.5 and 2.6 missing datetime.timedelta.total_seconds - res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 +def seconds_to_midnight(utc=None): + if utc is None: + now = datetime.datetime.today() + else: + now = datetime.datetime.utcnow() + datetime.timedelta(hours=utc) - return int(res) + midnight = now.replace(hour=0, minute=10, second=0, microsecond=0) + datetime.timedelta(days=1) + + return (midnight - now).seconds def replace_patterns(string, ruleslist): @@ -145,8 +172,8 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" - __type__ = "hoster" - __version__ = "0.30" + __type__ = "plugin" + __version__ = "0.37" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -165,6 +192,11 @@ class Plugin(object): self.init() + def __repr__(self): + return "<%(type)s %(name)s>" % {'type': self.__type__.capitalize(), + 'name': self.__name__} + + def _init(self, core): self.pyload = core self.info = {} #: Provide information in dict here @@ -180,33 +212,39 @@ class Plugin(object): def _log(self, level, plugintype, pluginname, messages): log = getattr(self.pyload.log, level) - msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in messages if a)) - log("%(plugintype)s %(pluginname)s%(id)s: %(msg)s" + msg = u" | ".join(decode(a).strip() for a in messages if a) + log("%(plugintype)s %(pluginname)s: %(msg)s" % {'plugintype': plugintype.upper(), 'pluginname': pluginname, - 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", 'msg' : msg}) def log_debug(self, *args): - if self.pyload.debug: - return self._log("debug", self.__type__, self.__name__, args) + if not self.pyload.debug: + return + self._log("debug", self.__type__, self.__name__, args) def log_info(self, *args): - return self._log("info", self.__type__, self.__name__, args) + self._log("info", self.__type__, self.__name__, args) def log_warning(self, *args): - return self._log("warning", self.__type__, self.__name__, args) + self._log("warning", self.__type__, self.__name__, args) + if self.pyload.debug: + traceback.print_exc() def log_error(self, *args): - return self._log("error", self.__type__, self.__name__, args) + self._log("error", self.__type__, self.__name__, args) + if self.pyload.debug: + traceback.print_exc() def log_critical(self, *args): return self._log("critical", self.__type__, self.__name__, args) + if self.pyload.debug: + traceback.print_exc() def set_permissions(self, path): @@ -287,21 +325,10 @@ class Plugin(object): self.pyload.db.delStorage(self.__name__, key) - def fail(self, reason): + def fail(self, msg): """ - Fail and give reason + Fail and give msg """ - raise Fail(encode(reason)) #@TODO: Remove `encode` in 0.4.10 - - - def error(self, reason="", type=_("Parse")): - if not reason: - type = _("Unknown") - - msg = _("%s error") % type.strip().capitalize() if type else _("Error") - msg += (": %s" % reason.strip()) if reason else "" - msg += _(" | Plugin may be out of date") - raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 @@ -318,14 +345,6 @@ class Plugin(object): :param decode: Wether to decode the output according to http header, should be True in most cases :return: Loaded content """ - if hasattr(self, 'pyfile') and self.pyfile.abort: - self.abort() - - url = fixurl(url) - - if not url or not isinstance(url, basestring): - self.fail(_("No url given")) - if self.pyload.debug: self.log_debug("LOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) @@ -345,7 +364,7 @@ class Plugin(object): #@TODO: Move to network in 0.4.10 if isinstance(decode, basestring): - res = decode(res, decode) + res = sys.modules[self.__name__].decode(res, decode) #@TODO: See #1787, use utils.decode() in 0.4.10 if self.pyload.debug: frame = inspect.currentframe() @@ -391,6 +410,7 @@ class Plugin(object): """ try: self.req.close() + except Exception: pass diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index 5811c28de..b79256536 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -10,7 +10,7 @@ from module.utils import fs_encode, save_join as fs_join class SevenZip(UnRar): __name__ = "SevenZip" - __version__ = "0.14" + __version__ = "0.15" __status__ = "testing" __description__ = """7-Zip extractor plugin""" @@ -55,42 +55,28 @@ class SevenZip(UnRar): return True - def verify(self, password): + def verify(self, password=None): #: 7z can't distinguish crc and pw error in test - p = self.call_cmd("l", "-slt", fs_encode(self.filename)) + p = self.call_cmd("l", "-slt", self.target) out, err = p.communicate() if self.re_wrongpwd.search(out): raise PasswordError - if self.re_wrongpwd.search(err): + elif self.re_wrongpwd.search(err): raise PasswordError - if self.re_wrongcrc.search(err): - raise CRCError(err) - - - - def check(self, password): - p = self.call_cmd("l", "-slt", fs_encode(self.filename)) - out, err = p.communicate() - - #: Check if output or error macthes the 'wrong password'-Regexp - if self.re_wrongpwd.search(out): - raise PasswordError - - if self.re_wrongcrc.search(out): + elif self.re_wrongcrc.search(out): raise CRCError(_("Header protected")) - - def repair(self): - return False + elif self.re_wrongcrc.search(err): + raise CRCError(err) def extract(self, password=None): command = "x" if self.fullpath else "e" - p = self.call_cmd(command, '-o' + self.out, fs_encode(self.filename), password=password) + p = self.call_cmd(command, '-o' + self.out, self.target, password=password) renice(p.pid, self.renice) @@ -117,7 +103,7 @@ class SevenZip(UnRar): def list(self, password=None): command = "l" if self.fullpath else "l" - p = self.call_cmd(command, fs_encode(self.filename), password=password) + p = self.call_cmd(command, self.target, password=password) out, err = p.communicate() if "Can not open" in err: diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 6a3f91a5b..8c5d3599d 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -4,17 +4,16 @@ import re from module.plugins.internal.Crypter import Crypter from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookie, set_cookies -from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.60" + __version__ = "0.62" __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), #: Overrides pyload.config['general']['folder_per_package'] + __config__ = [("use_subfolder" , "bool", "Save package to subfolder" , True), ("subfolder_per_pack", "bool", "Create a subfolder for each package", True)] __description__ = """Simple decrypter plugin""" @@ -90,6 +89,11 @@ class SimpleCrypter(Crypter, SimpleHoster): self.log_error(_("Too many redirects")) + def prepare(self): + self.links = [] + return super(SimpleCrypter, self).prepare() + + def decrypt(self, pyfile): self.prepare() self.check_info() #@TODO: Remove in 0.4.10 @@ -132,7 +136,8 @@ class SimpleCrypter(Crypter, SimpleHoster): def handle_pages(self, pyfile): try: pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) - except Exception: + + except AttributeError: pages = 1 for p in xrange(2, pages + 1): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 69f88081a..0f030e000 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -5,13 +5,12 @@ from __future__ import with_statement import os import re import time -import urlparse from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, encode, fixurl, replace_patterns, seconds_to_midnight, set_cookie, set_cookies +from module.plugins.internal.Plugin import Fail, encode, parse_name, replace_patterns, seconds_to_midnight, set_cookie, set_cookies from module.utils import fixup, fs_encode, parseFileSize as parse_size @@ -22,12 +21,13 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.81" + __version__ = "1.86" __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [("use_premium", "bool", "Use premium account if available" , True), - ("fallback" , "bool", "Fallback to free download if premium fails", True)] + __config__ = [("use_premium" , "bool", "Use premium account if available" , True), + ("premium_fallback", "bool", "Fallback to free download if premium fails", True), + ("chk_filesize" , "bool", "Check file size" , True)] __description__ = """Simple hoster plugin""" __license__ = "GPLv3" @@ -90,7 +90,7 @@ class SimpleHoster(Hoster): LINK_PREMIUM_PATTERN: (optional) group(1) should be the direct link for premium download example: LINK_PREMIUM_PATTERN = r'<div class="link"><a href="(.+?)"' """ - NAME_REPLACEMENTS = [("&#?\w+;", fixup)] + NAME_REPLACEMENTS = [] SIZE_REPLACEMENTS = [] URL_REPLACEMENTS = [] @@ -124,7 +124,7 @@ class SimpleHoster(Hoster): try: info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: Pattern groups will be saved here - except Exception: + except AttributeError: info['pattern'] = {} if not html and not online: @@ -174,8 +174,8 @@ class SimpleHoster(Hoster): info['status'] = 2 if 'N' in info['pattern']: - info['name'] = replace_patterns(fixurl(info['pattern']['N']), - cls.NAME_REPLACEMENTS) + name = replace_patterns(info['pattern']['N'], cls.NAME_REPLACEMENTS) + info['name'] = parse_name(name) if 'S' in info['pattern']: size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info['pattern'] else info['pattern']['S'], @@ -201,19 +201,15 @@ class SimpleHoster(Hoster): def prepare(self): - self.pyfile.error = "" #@TODO: Remove in 0.4.10 - self.html = "" #@TODO: Recheck in 0.4.10 - self.link = "" #@TODO: Recheck in 0.4.10 - self.last_download = "" - self.direct_dl = False - self.leech_dl = False - - if not self.get_config('use_premium', True): + self.link = "" + self.direct_dl = False + self.leech_dl = False + + if not self.get_config('use_premium', True) and self.premium: self.restart(nopremium=True) if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) - self.LOGIN_ACCOUNT = True if self.LOGIN_ACCOUNT and not self.account: self.fail(_("Required account not found")) @@ -294,7 +290,7 @@ class SimpleHoster(Hoster): self.check_file() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 - if self.get_config('fallback', True) and self.premium: + if self.get_config('premium_fallback', True) and self.premium: self.log_warning(_("Premium download failed"), e) self.restart(nopremium=True) @@ -307,17 +303,18 @@ class SimpleHoster(Hoster): if self.captcha.task and not self.last_download: self.captcha.invalid() - self.retry(10, reason=_("Wrong captcha")) + self.retry(10, msg=_("Wrong captcha")) - # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size - # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, - file_size=self.info['size'] if 'size' in self.info else 0, - size_tolerance=10485760, - delete=False): #@TODO: Make `delete` settable in 0.4.10 + delete=True): self.error(_("Empty file")) else: + if self.get_config('chk_filesize', False) and 'size' in self.info: + # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size + # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB + self.check_filesize(self.info['size'], size_tolerance=10485760) + self.log_debug("Using default check rules...") for r, p in self.FILE_ERRORS: errmsg = self.check_download({r: re.compile(p)}) @@ -326,12 +323,13 @@ class SimpleHoster(Hoster): try: errmsg += " | " + self.last_check.group(1).strip() + except Exception: pass self.log_warning(_("Check result: ") + errmsg, _("Waiting 1 minute and retry")) self.wantReconnect = True - self.retry(wait_time=60, reason=errmsg) + self.retry(delay=60, msg=errmsg) else: if self.CHECK_FILE: self.log_debug("Using custom check rules...") @@ -340,7 +338,7 @@ class SimpleHoster(Hoster): self.check_errors() self.log_info(_("No errors found")) - self.pyfile.error = "" + self.pyfile.error = "" #@TODO: Recheck in 0.4.10 def check_errors(self): @@ -362,14 +360,15 @@ class SimpleHoster(Hoster): m = re.search(self.DL_LIMIT_PATTERN, self.html) try: errmsg = m.group(1).strip() - except Exception: + + except AttributeError: errmsg = m.group(0).strip() self.info['error'] = re.sub(r'<.*?>', " ", errmsg) self.log_warning(self.info['error']) if re.search('da(il)?y|today', errmsg, re.I): - wait_time = seconds_to_midnight(gmt=2) + wait_time = seconds_to_midnight() else: wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) @@ -385,7 +384,8 @@ class SimpleHoster(Hoster): if m: try: errmsg = m.group(1).strip() - except Exception: + + except AttributeError: errmsg = m.group(0).strip() self.info['error'] = re.sub(r'<.*?>', " ", errmsg) @@ -393,7 +393,7 @@ class SimpleHoster(Hoster): if re.search('limit|wait|slot', errmsg, re.I): if re.search("da(il)?y|today", errmsg): - wait_time = seconds_to_midnight(gmt=2) + wait_time = seconds_to_midnight() else: wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec|)', errmsg, re.I)) @@ -406,7 +406,7 @@ class SimpleHoster(Hoster): elif re.search('captcha|code', errmsg, re.I): self.captcha.invalid() - self.retry(10, reason=_("Wrong captcha")) + self.retry(10, msg=_("Wrong captcha")) elif re.search('countdown|expired', errmsg, re.I): self.retry(10, 60, _("Link expired")) @@ -421,23 +421,22 @@ class SimpleHoster(Hoster): self.offline() elif re.search('filename', errmsg, re.I): - url_p = urlparse.urlparse(self.pyfile.url) - self.pyfile.url = "%s://%s/%s" % (url_p.scheme, url_p.netloc, url_p.path.split('/')[0]) - self.retry(1, reason=_("Wrong url")) + self.fail(_("Wrong url")) elif re.search('premium', errmsg, re.I): self.fail(_("File can be downloaded by premium users only")) else: self.wantReconnect = True - self.retry(wait_time=60, reason=errmsg) + self.retry(delay=60, msg=errmsg) elif hasattr(self, 'WAIT_PATTERN'): m = re.search(self.WAIT_PATTERN, self.html) if m: try: waitmsg = m.group(1).strip() - except Exception: + + except AttributeError: waitmsg = m.group(0).strip() wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in @@ -483,8 +482,8 @@ class SimpleHoster(Hoster): self.log_debug("Previous file info: %s" % old_info) try: - url = self.info['url'].strip() - name = self.info['name'].strip() + url = self.info['url'] + name = self.info['name'] except KeyError: pass diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 0386991d9..88c490750 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.25" + __version__ = "1.26" __status__ = "testing" __description__ = """Rar extractor plugin""" @@ -84,20 +84,8 @@ class UnRar(Extractor): return True if cls.re_multipart.search(filename) else False - def verify(self, password): - p = self.call_cmd("t", "-v", fs_encode(self.filename), password=password) - self._progress(p) - err = p.stderr.read().strip() - - if self.re_wrongpwd.search(err): - raise PasswordError - - if self.re_wrongcrc.search(err): - raise CRCError(err) - - - def check(self, password): - p = self.call_cmd("l", "-v", fs_encode(self.filename), password=password) + def verify(self, password=None): + p = self.call_cmd("l", "-v", self.target, password=password) out, err = p.communicate() if self.re_wrongpwd.search(err): @@ -113,13 +101,28 @@ class UnRar(Extractor): def repair(self): - p = self.call_cmd("rc", fs_encode(self.filename)) + p = self.call_cmd("rc", self.target) #: Communicate and retrieve stderr self._progress(p) err = p.stderr.read().strip() + if err or p.returncode: - return False + p = self.call_cmd("r", self.target) + + # communicate and retrieve stderr + self._progress(p) + err = p.stderr.read().strip() + + if err or p.returncode: + return False + + else: + dir = os.path.dirname(filename) + name = re_filefixed.search(out).group(1) + + self.filename = os.path.join(dir, name) + return True @@ -145,7 +148,7 @@ class UnRar(Extractor): def extract(self, password=None): command = "x" if self.fullpath else "e" - p = self.call_cmd(command, fs_encode(self.filename), self.out, password=password) + p = self.call_cmd(command, self.target, self.out, password=password) renice(p.pid, self.renice) @@ -169,7 +172,7 @@ class UnRar(Extractor): self.files = self.list(password) - def get_delete_files(self): + def items(self): dir, name = os.path.split(self.filename) #: Actually extracted file @@ -185,7 +188,7 @@ class UnRar(Extractor): def list(self, password=None): command = "vb" if self.fullpath else "lb" - p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password) + p = self.call_cmd(command, "-v", self.target, password=password) out, err = p.communicate() if "Cannot open" in err: diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 9a01611bf..ac197a80d 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -12,7 +12,7 @@ from module.utils import fs_encode class UnZip(Extractor): __name__ = "UnZip" - __version__ = "1.15" + __version__ = "1.16" __status__ = "testing" __description__ = """Zip extractor plugin""" @@ -30,17 +30,13 @@ class UnZip(Extractor): def list(self, password=None): - with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: + with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z: z.setpassword(password) return z.namelist() - def check(self, password): - pass - - - def verify(self): - with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: + def verify(self, password=None): + with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z: badfile = z.testzip() if badfile: @@ -51,7 +47,7 @@ class UnZip(Extractor): def extract(self, password=None): try: - with zipfile.ZipFile(fs_encode(self.filename), 'r', allowZip64=True) as z: + with zipfile.ZipFile(self.target, 'r', allowZip64=True) as z: z.setpassword(password) badfile = z.testzip() diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 5a4cc35fb..bb5cbcf50 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -4,6 +4,7 @@ import re import time import urlparse +from module.common.json_layer import json_loads from module.plugins.internal.Account import Account from module.plugins.internal.Plugin import parse_html_form, set_cookie @@ -11,7 +12,7 @@ from module.plugins.internal.Plugin import parse_html_form, set_cookie class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.43" + __version__ = "0.46" __status__ = "testing" __description__ = """XFileSharing account plugin""" @@ -39,7 +40,7 @@ class XFSAccount(Account): LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|account was banned|Error<' - def parse_info(self, user, password, data, req): + def grab_info(self, user, password, data, req): validuntil = None trafficleft = None leechtraffic = None @@ -144,13 +145,13 @@ class XFSAccount(Account): self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN if self.COOKIES: - if isinstance(self.COOKIES, list) and not self.COOKIES.count((self.HOSTER_DOMAIN, "lang", "english")): + if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) else: set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") if not self.HOSTER_URL: - self.login_fail(_("Missing HOSTER_URL")) + self.fail_login(_("Missing HOSTER_URL")) else: self.HOSTER_URL = self.HOSTER_URL.rstrip('/') + "/" @@ -174,5 +175,13 @@ class XFSAccount(Account): html = self.load(url, post=inputs, cookies=self.COOKIES) - if re.search(self.LOGIN_FAIL_PATTERN, html): - self.login_fail() + try: + json = json_loads(html) + + except ValueError: + if re.search(self.LOGIN_FAIL_PATTERN, html): + self.fail_login() + + else: + if not 'success' in json or not json['success']: + self.fail_login() diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 4c059d647..12a48d4a3 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -7,7 +7,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.13" + __version__ = "0.14" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -42,7 +42,7 @@ class XFSCrypter(SimpleCrypter): self.fail(_("Missing HOSTER_DOMAIN")) if self.COOKIES: - if isinstance(self.COOKIES, list) and not self.COOKIES.count((self.HOSTER_DOMAIN, "lang", "english")): + if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) else: set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 5e0830dc6..729c9a0ee 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.57" + __version__ = "0.60" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -73,7 +73,7 @@ class XFSHoster(SimpleHoster): self.fail(_("Missing HOSTER_DOMAIN")) if self.COOKIES: - if isinstance(self.COOKIES, list) and not self.COOKIES.count((self.HOSTER_DOMAIN, "lang", "english")): + if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) else: set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") @@ -150,7 +150,7 @@ class XFSHoster(SimpleHoster): action, inputs = self.parse_html_form('F1') if not inputs: - self.retry(reason=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) + self.retry(msg=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) self.log_debug(inputs) @@ -163,7 +163,7 @@ class XFSHoster(SimpleHoster): self.retry(20, 3 * 60, _("Can not leech file")) elif 'today' in stmsg: - self.retry(wait_time=seconds_to_midnight(gmt=2), reason=_("You've used all Leech traffic today")) + self.retry(delay=seconds_to_midnight(), msg=_("You've used all Leech traffic today")) else: self.fail(stmsg) @@ -188,7 +188,7 @@ class XFSHoster(SimpleHoster): if not inputs: action, inputs = self.parse_html_form('F1') if not inputs: - self.retry(reason=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) + self.retry(msg=self.info['error'] if 'error' in self.info else _("TEXTAREA F1 not found")) self.log_debug(inputs) @@ -244,7 +244,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) - except Exception: + except AttributeError: captcha_key = recaptcha.detect_key() else: @@ -258,7 +258,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) - except Exception: + except AttributeError: captcha_key = solvemedia.detect_key() else: |