diff options
| -rw-r--r-- | module/plugins/internal/Account.py | 10 | ||||
| -rw-r--r-- | module/plugins/internal/Captcha.py | 26 | ||||
| -rw-r--r-- | module/plugins/internal/CaptchaService.py | 6 | ||||
| -rw-r--r-- | module/plugins/internal/Container.py | 18 | ||||
| -rw-r--r-- | module/plugins/internal/Extractor.py | 7 | ||||
| -rw-r--r-- | module/plugins/internal/Hoster.py | 67 | ||||
| -rw-r--r-- | module/plugins/internal/MultiHoster.py | 50 | ||||
| -rw-r--r-- | module/plugins/internal/OCR.py | 8 | ||||
| -rw-r--r-- | module/plugins/internal/Plugin.py | 19 | ||||
| -rw-r--r-- | module/plugins/internal/SimpleCrypter.py | 74 | ||||
| -rw-r--r-- | module/plugins/internal/SimpleHoster.py | 163 | ||||
| -rw-r--r-- | module/plugins/internal/XFSHoster.py | 8 |
12 files changed, 231 insertions, 225 deletions
diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index b59a59932..f1a30a9cb 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -14,7 +14,7 @@ from module.utils import compare_time, lock, parseFileSize as parse_size class Account(Plugin): __name__ = "Account" __type__ = "account" - __version__ = "0.05" + __version__ = "0.06" __status__ = "testing" __description__ = """Base account plugin""" @@ -66,7 +66,7 @@ class Account(Plugin): traceback.print_exc() else: - res = True + res = info['login']['valid'] = True finally: if self.req: @@ -96,7 +96,7 @@ class Account(Plugin): @lock def add(self, user, password=None, options={}): if user not in self.info: - self.info[user] = {'login': {'valid': True, 'password': password or "", 'timestamp': 0}, #@NOTE: Do not remove `'valid': True` in 0.4.9 or accounts will not login + self.info[user] = {'login': {'valid': None, 'password': password or "", 'timestamp': 0}, 'data' : {'options': options, 'timestamp': 0}} self._login(user) return True @@ -171,8 +171,6 @@ class Account(Plugin): :param reload: reloads cached account information :return: dictionary with information """ - traceback.print_exc() ###################### - if user not in self.info: self.log_error(_("User %s not found while retrieving account info") % user) return @@ -312,7 +310,7 @@ class Account(Plugin): def can_use(self): - return self.select() is not (None, None) + return self.select() != (None, None) def parse_traffic(self, value, unit=None): #: Return kilobytes diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index ec618e353..32d882fac 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -10,7 +10,7 @@ from module.plugins.internal.Plugin import Plugin class Captcha(Plugin): __name__ = "Captcha" __type__ = "captcha" - __version__ = "0.31" + __version__ = "0.32" __status__ = "testing" __description__ = """Base anti-captcha plugin""" @@ -20,9 +20,8 @@ class Captcha(Plugin): def __init__(self, plugin): #@TODO: Pass pyfile instead plugin, so store plugin's html in its associated pyfile as data self.pyload = plugin.pyload - self.info = {} #: Provide information in dict here - self.plugin = plugin + self.info = {} #: Provide information in dict here self.task = None #: captchaManager task self.init() @@ -35,6 +34,10 @@ class Captcha(Plugin): pass + def _log(self, level, args): + return self.plugin._log(level, (self.__name__,) + args) + + def recognize(self, image): """ Extend to build your custom anti-captcha ocr @@ -43,17 +46,17 @@ class Captcha(Plugin): def decrypt(self, url, get={}, post={}, ref=False, cookies=False, decode=False, - input_type='png', output_type='textual', ocr=True): + input_type='jpg', output_type='textual', ocr=True): img = self.load(url, get=get, post=post, ref=ref, cookies=cookies, decode=decode) return self._decrypt(img, input_type, output_type, ocr) #@TODO: Definitely dhoose a better name for this method! - def _decrypt(self, raw, input_type='png', output_type='textual', ocr=None): + def _decrypt(self, raw, input_type='jpg', output_type='textual', ocr=None): """ Loads a captcha and decrypts it with ocr, plugin, user input - :param url: url of captcha image + :param raw: image raw data :param get: get part for request :param post: post part for request :param cookies: True if cookies should be enabled @@ -75,7 +78,7 @@ class Captcha(Plugin): OCR = self.pyload.pluginManager.loadClass("captcha", ocr) #: Rename `captcha` to `ocr` in 0.4.10 if self.plugin.pyfile.abort: - self.abort() + self.plugin.abort() result = OCR(self.plugin).recognize(tmp_img.name) @@ -86,12 +89,12 @@ class Captcha(Plugin): captchaManager = self.pyload.captchaManager try: - self.task = captchaManager.newTask(img, input_type, tmp_img.name, output_type) + self.task = captchaManager.newTask(raw, input_type, tmp_img.name, output_type) captchaManager.handleCaptcha(self.task) while self.task.isWaiting(): if self.plugin.pyfile.abort: - self.abort() + self.plugin.abort() time.sleep(1) finally: captchaManager.removeTask(self.task) @@ -102,12 +105,13 @@ class Captcha(Plugin): elif not self.task.result: self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) - result = task.result - self.log_debug("Received captcha result: %s" % result) #@TODO: Remove from here? + result = self.task.result + self.log_info(_("Captcha result: ") + result) #@TODO: Remove from here? if not self.pyload.debug: try: os.remove(tmp_img.name) + except OSError, e: self.log_warning(_("Error removing: %s") % tmp_img.name, e) traceback.print_exc() diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 1a73ce5d1..20dc60427 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -6,7 +6,7 @@ from module.plugins.internal.Captcha import Captcha class CaptchaService(Captcha): __name__ = "CaptchaService" __type__ = "captcha" - __version__ = "0.31" + __version__ = "0.32" __status__ = "testing" __description__ = """Base anti-captcha service plugin""" @@ -18,10 +18,6 @@ class CaptchaService(Captcha): self.key = None #: Last key detected - def _log(self, level, args): - return self.plugin._log(level, (self.__name__,) + args) - - #@TODO: Recheck in 0.4.10 def retrieve_key(self, data): if self.detect_key(data): diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index c7a6828c7..ec2d429dd 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -13,7 +13,7 @@ from module.utils import save_join as fs_join class Container(Crypter): __name__ = "Container" __type__ = "container" - __version__ = "0.05" + __version__ = "0.06" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -24,18 +24,22 @@ class Container(Crypter): __authors__ = [("mkaay", "mkaay@mkaay.de")] - def preprocessing(self, thread): + def process(self, pyfile): """ - Prepare + Main method """ - self.setup() - self.thread = thread - self._load2disk() - self.decrypt(self.pyfile) + self.decrypt(pyfile) + self.delete_tmp() + if self.urls: + self._generate_packages() + + elif not self.packages: + self.error(_("No link grabbed"), "decrypt") + self._create_packages() diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index 668207ea3..cf9a1f6e3 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -22,7 +22,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.26" + __version__ = "0.27" __status__ = "testing" __description__ = """Base extractor plugin""" @@ -87,6 +87,7 @@ class Extractor(Plugin): Initialize extractor for specific file """ self.pyload = plugin.pyload + self.plugin = plugin self.info = {} #: Provide information in dict here self.filename = filename @@ -112,6 +113,10 @@ class Extractor(Plugin): pass + def _log(self, level, args): + return self.plugin._log(level, (self.__name__,) + args) + + def check(self): """ Quick Check by listing content of archive. diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 10cab5616..c35178547 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -47,7 +47,7 @@ def create_getInfo(klass): class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.05" + __version__ = "0.06" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -179,6 +179,7 @@ class Hoster(Plugin): self.pyfile.setStatus("starting") + self.log_debug("PROCESS URL " + self.pyfile.url) return self.process(self.pyfile) @@ -195,19 +196,10 @@ class Hoster(Plugin): return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) - def reset_account(self): - """ - Don't use account and retry download - """ - self.account = None - self.req = self.pyload.requestFactory.getRequest(self.__name__) - self.retry() - - def set_reconnect(self, reconnect): reconnect = bool(reconnect) - self.log_info(_("Reconnect: %s") % reconnect) + self.log_info(_("RECONNECT ") + ("enabled" if reconnect else "disabled")) self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) self.wantReconnect = reconnect @@ -223,7 +215,7 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_info(_("Wait: %d seconds") % wait_time) + self.log_info(_("WAIT %d seconds") % wait_time) self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until @@ -249,14 +241,16 @@ class Hoster(Plugin): status = pyfile.status #@NOTE: Remove in 0.4.10 pyfile.setStatus("waiting") - if self.account: - self.log_debug("Ignore reconnection due account logged") + if not self.wantReconnect or self.account: + if self.account: + self.log_warning("Ignore reconnection due logged account") while pyfile.waitUntil > time.time(): if pyfile.abort: self.abort() time.sleep(3) + else: while pyfile.waitUntil > time.time(): if pyfile.abort: @@ -266,7 +260,7 @@ class Hoster(Plugin): self.waiting = False self.wantReconnect = False raise Reconnect - + self.thread.m.reconnecting.wait(3) time.sleep(3) @@ -321,7 +315,7 @@ class Hoster(Plugin): self.retries[id] = 0 if 0 < max_tries <= self.retries[id]: - self.fail(reason or _("Max retries reached"), _("retry")) + self.fail(reason or _("Max retries reached")) self.wait(wait_time, False) @@ -329,6 +323,21 @@ class Hoster(Plugin): raise Retry(reason) + def restart(self, reason=None, reset=False): + if not reason: + reason = _("Fallback to free download") if reset else _("Restart") + + if reset: + if not self.premium: + return + + self.premium = False + self.account = None + self.req = self.pyload.requestFactory.getRequest(self.__name__) + + raise Retry(reason) + + def fixurl(self, url): url = _fixurl(url) @@ -362,7 +371,8 @@ class Hoster(Plugin): self.fail(_("No url given")) if self.pyload.debug: - self.log_debug("DOWNLOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + self.log_debug("DOWNLOAD URL " + url, + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) self.captcha.correct() self.check_for_same_files() @@ -432,7 +442,7 @@ class Hoster(Plugin): return self.last_download - def check_download(self, rules, delete=True, file_size=0, size_tolerance=1000, read_size=100000): + def check_download(self, rules, delete=False, file_size=0, size_tolerance=2048, read_size=100000): """ Checks the content of the last downloaded file, re match is saved to `lastCheck` @@ -444,14 +454,14 @@ class Hoster(Plugin): :return: dictionary key of the first rule that matched """ do_delete = False - lastDownload = fs_encode(self.last_download) + last_download = fs_encode(self.last_download) - if not self.last_download or not os.path.exists(lastDownload): + if not self.last_download or not os.path.exists(last_download): self.last_download = "" self.fail(self.pyfile.error or _("No file downloaded")) try: - download_size = os.stat(lastDownload).st_size + download_size = os.stat(last_download).st_size if download_size < 1: do_delete = True @@ -462,14 +472,13 @@ class Hoster(Plugin): if diff > size_tolerance: do_delete = True - self.fail(_("File size mismatch")) + self.fail(_("File size mismatch | Expected file size: %s | Downloaded file size: %s") + % (file_size, download_size)) elif diff != 0: self.log_warning(_("File size is not equal to expected size")) - self.log_debug("Download Check triggered") - - with open(lastDownload, "rb") as f: + with open(last_download, "rb") as f: content = f.read(read_size) #: Produces encoding errors, better log to other file in the future? @@ -489,12 +498,16 @@ class Hoster(Plugin): finally: if delete and do_delete: try: - os.remove(lastDownload) + os.remove(last_download) + except OSError, e: - self.log_warning(_("Error removing: %s") % lastDownload, e) + self.log_warning(_("Error removing: %s") % last_download, e) if self.pyload.debug: traceback.print_exc() + else: + self.log_info(_("File deleted")) + def direct_link(self, url, follow_location=None): link = "" diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index fe6fa29d6..b569b04c3 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.43" + __version__ = "0.44" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -21,48 +21,49 @@ class MultiHoster(SimpleHoster): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + LEECH_HOSTER = False LOGIN_ACCOUNT = True def setup(self): self.chunk_limit = 1 - self.multiDL = bool(self.account) + self.multiDL = bool(self.account) self.resume_download = self.premium def prepare(self): - self.html = "" - self.link = "" #@TODO: Move to Hoster in 0.4.10 - self.direct_dl = False #@TODO: Move to Hoster in 0.4.10 + #@TODO: Recheck in 0.4.10 + plugin = self.pyload.pluginManager.hosterPlugins[self.__name__] + name = plugin['name'] + module = plugin['module'] + klass = getattr(module, name) - if not self.get_config('use_premium', True): - self.retry_free() - - if self.LOGIN_ACCOUNT and not self.account: - self.fail(_("Required account not found")) - - self.req.setOption("timeout", 120) - - if isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) + self.get_info = klass.get_info if self.DIRECT_LINK is None: - self.direct_dl = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) + direct_dl = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) else: - self.direct_dl = self.DIRECT_LINK + direct_dl = self.DIRECT_LINK - self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) + super(MultiHoster, self).prepare() + + self.direct_dl = direct_dl def process(self, pyfile): try: self.prepare() + self.check_info() #@TODO: Remove in 0.4.10 if self.direct_dl: - self.check_info() - self.log_debug("Looking for direct download link...") + self.log_info(_("Looking for direct download link...")) self.handle_direct(pyfile) + if self.link or was_downloaded(): + self.log_info(_("Direct download link detected")) + else: + self.log_info(_("Direct download link not found")) + if not self.link and not self.last_download: self.preload() @@ -70,14 +71,17 @@ class MultiHoster(SimpleHoster): self.check_status(getinfo=False) if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): - self.log_debug("Handled as premium download") + self.log_info(_("Processing as premium download...")) self.handle_premium(pyfile) elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): - self.log_debug("Handled as free download") + self.log_info(_("Processing as free download...")) self.handle_free(pyfile) - self.download(self.link, ref=False, disposition=True) + if not self.last_download: + self.log_info(_("Downloading file...")) + self.download(self.link, ref=False, disposition=True) + self.check_file() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 diff --git a/module/plugins/internal/OCR.py b/module/plugins/internal/OCR.py index 9896837b5..37a33206f 100644 --- a/module/plugins/internal/OCR.py +++ b/module/plugins/internal/OCR.py @@ -21,7 +21,7 @@ from module.utils import save_join as fs_join class OCR(Plugin): __name__ = "OCR" __type__ = "ocr" - __version__ = "0.13" + __version__ = "0.14" __status__ = "testing" __description__ = """OCR base plugin""" @@ -31,7 +31,9 @@ class OCR(Plugin): def __init__(self, plugin): self.pyload = plugin.pyload + self.plugin = plugin self.info = {} #: Provide information in dict here + self.init() @@ -42,6 +44,10 @@ class OCR(Plugin): pass + def _log(self, level, args): + return self.plugin._log(level, (self.__name__,) + args) + + def load_image(self, image): self.image = Image.open(image) self.pixels = self.image.load() diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 7b3d03752..f88660f24 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -5,7 +5,6 @@ from __future__ import with_statement import inspect import os import re -import traceback import urllib from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 @@ -126,7 +125,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.13" + __version__ = "0.14" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -155,11 +154,11 @@ class Plugin(object): def _log(self, level, args): log = getattr(self.pyload.log, level) - msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in args if a)) #@NOTE: `fs_encode` -> `encode` in 0.4.10 - log("%(type)s %(plugin)s%(id)s: %(msg)s" % {'type': self.__type__.upper(), - 'plugin': self.__name__, - 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", - 'msg' : msg or _(level.upper() + " MARK")}) + msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in args if a)) + log("%(type)s %(plugin)s%(id)s : %(msg)s" % {'type': self.__type__.upper(), + 'plugin': self.__name__, + 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", + 'msg' : msg or "---------- MARK ----------"}) def log_debug(self, *args): @@ -205,8 +204,7 @@ class Plugin(object): return self.pyload.config.getPlugin(plugin or self.__name__, option) except KeyError: - self.log_warning(_("Config option or plugin not found")) - traceback.print_exc() + self.log_warning(_("Config option `%s` not found, use default `%s`") % (option, default or None)) return default @@ -271,7 +269,8 @@ class Plugin(object): self.fail(_("No url given")) if self.pyload.debug: - self.log_debug("LOAD URL " + url, *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + self.log_debug("LOAD URL " + url, + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) if req is None: if hasattr(self, "req"): diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 308ba2966..33ebd8cac 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.56" + __version__ = "0.57" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -51,6 +51,11 @@ class SimpleCrypter(Crypter, SimpleHoster): def load_page(self, page_n): return the html of the page number page_n """ + + DIRECT_LINK = True + LEECH_HOSTER = False + + #@TODO: Remove in 0.4.10 def init(self): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") @@ -64,27 +69,6 @@ class SimpleCrypter(Crypter, SimpleHoster): self.account = account - def prepare(self): - self.pyfile.error = "" #@TODO: Remove in 0.4.10 - - self.html = "" #@TODO: Recheck in 0.4.10 - self.link = "" #@TODO: Recheck in 0.4.10 - self.links = [] - - if self.LOGIN_PREMIUM and not self.premium: - self.fail(_("Required premium account not found")) - - if self.LOGIN_ACCOUNT and not self.account: - self.fail(_("Required account not found")) - - self.req.setOption("timeout", 120) - - if isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) - - self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - - def handle_direct(self, pyfile): for i in xrange(self.get_config("maxredirs", plugin="UserAgentSwitcher")): redirect = self.link or pyfile.url @@ -101,16 +85,19 @@ class SimpleCrypter(Crypter, SimpleHoster): def decrypt(self, pyfile): self.prepare() + self.check_info() #@TODO: Remove in 0.4.10 - self.log_debug("Looking for link redirect...") - self.handle_direct(pyfile) + if self.direct_dl: + self.log_debug(_("Looking for direct download link...")) + self.handle_direct(pyfile) - if self.link: - self.urls = [self.link] + if self.link or self.links or self.urls or self.packages: + self.log_info(_("Direct download link detected")) + else: + self.log_info(_("Direct download link not found")) - else: + if not (self.link or self.links or self.urls or self.packages): self.preload() - self.check_info() self.links = self.get_links() or list() @@ -119,33 +106,12 @@ class SimpleCrypter(Crypter, SimpleHoster): self.log_debug("Package has %d links" % len(self.links)) - if self.links: - self.packages = [(self.info['name'], self.links, self.info['folder'])] - - - def check_name_size(self, getinfo=True): - if not self.info or getinfo: - self.log_debug("File info (BEFORE): %s" % self.info) - self.info.update(self.get_info(self.pyfile.url, self.html)) - self.log_debug("File info (AFTER): %s" % self.info) - - try: - url = self.info['url'].strip() - name = self.info['name'].strip() - if name and name is not url: - self.pyfile.name = name - - except Exception: - pass - - try: - folder = self.info['folder'] = self.pyfile.name - - except Exception: - pass + if self.link: + self.urls.append(self.link) - self.log_debug("File name: %s" % self.pyfile.name, - "File folder: %s" % self.pyfile.name) + if self.links: + name = folder = pyfile.name + self.packages.append((name, self.links, folder)) def get_links(self): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index b063bcac2..e8281b943 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -24,7 +24,7 @@ statusMap = dict((v, k) for k, v in _statusMap.items()) class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.71" + __version__ = "1.72" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -107,7 +107,7 @@ class SimpleHoster(Hoster): DISPOSITION = True #: Set to True to use any content-disposition value in http header as file name LOGIN_ACCOUNT = False #: Set to True to require account login LOGIN_PREMIUM = False #: Set to True to require premium account login - MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handle_multi method) + LEECH_HOSTER = False #: Set to True to leech other hoster link (as defined in handle_multi method) TEXT_ENCODING = True #: Set to encoding name if encoding value in http header is not correct LINK_PATTERN = None @@ -203,18 +203,19 @@ class SimpleHoster(Hoster): def prepare(self): - self.pyfile.error = "" #@TODO: Remove in 0.4.10 - - self.html = "" #@TODO: Recheck in 0.4.10 - self.link = "" #@TODO: Recheck in 0.4.10 - self.direct_dl = False - self.multihost = False + self.pyfile.error = "" #@TODO: Remove in 0.4.10 + self.html = "" #@TODO: Recheck in 0.4.10 + self.link = "" #@TODO: Recheck in 0.4.10 + self.last_download = "" + self.direct_dl = False + self.leech_dl = False if not self.get_config('use_premium', True): self.retry_free() if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) + self.LOGIN_ACCOUNT = True if self.LOGIN_ACCOUNT and not self.account: self.fail(_("Required account not found")) @@ -231,18 +232,21 @@ class SimpleHoster(Hoster): if not hasattr(self, 'LINK_PREMIUM_PATTERN'): self.LINK_PREMIUM_PATTERN = self.LINK_PATTERN - if (self.MULTI_HOSTER + if (self.LEECH_HOSTER and (self.__pattern__ is not self.pyload.pluginManager.hosterPlugins[self.__name__]['pattern'] - or re.match(self.__pattern__, self.pyfile.url) is None)): - self.multihost = True - return + and re.match(self.__pattern__, self.pyfile.url) is None)): + self.leech_dl = True - if self.DIRECT_LINK is None: + if self.leech_dl: + self.direct_dl = False + + elif self.DIRECT_LINK is None: self.direct_dl = bool(self.account) else: self.direct_dl = self.DIRECT_LINK - self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) + if not self.leech_dl: + self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) def preload(self): @@ -255,33 +259,43 @@ class SimpleHoster(Hoster): def process(self, pyfile): try: self.prepare() - self.check_info() - - if self.direct_dl: - self.log_debug("Looking for direct download link...") - self.handle_direct(pyfile) + self.check_info() #@TODO: Remove in 0.4.10 - if self.multihost and not self.link and not self.last_download: - self.log_debug("Looking for leeched download link...") + if self.leech_dl: + self.log_info(_("Processing as debrid download...")) self.handle_multi(pyfile) + if not self.link and not was_downloaded(): + self.log_info(_("Failed to leech url")) + + else: + if not self.link and self.direct_dl and not self.last_download: + self.log_info(_("Looking for direct download link...")) + self.handle_direct(pyfile) + + if self.link or self.last_download: + self.log_info(_("Direct download link detected")) + else: + self.log_info(_("Direct download link not found")) + if not self.link and not self.last_download: - self.MULTI_HOSTER = False - self.retry(1, reason=_("Multi hoster fails")) + self.preload() + + if 'status' not in self.info or self.info['status'] is 3: #@TODO: Recheck in 0.4.10 + self.check_info() - if not self.link and not self.last_download: - self.preload() - self.check_info() + if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): + self.log_info(_("Processing as premium download...")) + self.handle_premium(pyfile) - if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): - self.log_debug("Handled as premium download") - self.handle_premium(pyfile) + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): + self.log_info(_("Processing as free download...")) + self.handle_free(pyfile) - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): - self.log_debug("Handled as free download") - self.handle_free(pyfile) + if not self.last_download: + self.log_info(_("Downloading file...")) + self.download(self.link, ref=False, disposition=self.DISPOSITION) - self.download(self.link, ref=False, disposition=self.DISPOSITION) self.check_file() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 @@ -292,25 +306,26 @@ class SimpleHoster(Hoster): elif self.get_config('fallback', True) and self.premium: self.log_warning(_("Premium download failed"), e) - self.retry_free() + self.restart(reset=True) else: raise Fail(err) def check_file(self): - lastDownload = fs_encode(self.last_download) + self.log_info(_("Checking file...")) if self.captcha.task and not self.last_download: self.captcha.invalid() self.retry(10, reason=_("Wrong captcha")) elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, - file_size=self.info['size']): + file_size=self.info['size'] if 'size' in self.info else 0, + delete=True): self.error(_("Empty file")) else: - self.log_debug("Checking last downloaded file with built-in rules") + self.log_debug("Using default check rules...") for r, p in self.FILE_ERRORS: errmsg = self.check_download({r: re.compile(p)}) if errmsg is not None: @@ -326,12 +341,13 @@ class SimpleHoster(Hoster): self.retry(wait_time=60, reason=errmsg) else: if self.CHECK_FILE: - self.log_debug("Checking last downloaded file with custom rules") - with open(lastDownload, "rb") as f: + self.log_debug("Using custom check rules...") + with open(fs_encode(self.last_download), "rb") as f: self.html = f.read(50000) #@TODO: Recheck in 0.4.10 self.check_errors() - self.log_debug("No file errors found") + self.log_debug("No errors found") + self.pyfile.error = "" def check_errors(self): @@ -440,13 +456,14 @@ class SimpleHoster(Hoster): def check_status(self, getinfo=True): if not self.info or getinfo: - self.log_debug("Update file info...") - self.log_debug("Previous file info: %s" % self.info) + self.log_info(_("Updating file info...")) + old_info = self.info.copy() self.info.update(self.get_info(self.pyfile.url, self.html)) - self.log_debug("Current file info: %s" % self.info) + self.log_debug("File info: %s" % self.info) + self.log_debug("Previous file info: %s" % old_info) try: - status = self.info['status'] + status = self.info['status'] or None if status == 1: self.offline() @@ -455,40 +472,49 @@ class SimpleHoster(Hoster): self.temp_offline() elif status == 8: - self.fail(self.info['error'] if 'error' in self.info else _("Failed")) + if 'error' in self.info: + self.fail(self.info['error']) + else: + self.fail(_("File status: " + statusMap[status])) finally: - self.log_debug("File status: %s" % statusMap[status]) + self.log_info(_("File status: ") + (statusMap[status] if status else _("Unknown"))) def check_name_size(self, getinfo=True): if not self.info or getinfo: - self.log_debug("Update file info...") - self.log_debug("Previous file info: %s" % self.info) + self.log_info(_("Updating file info...")) + old_info = self.info.copy() self.info.update(self.get_info(self.pyfile.url, self.html)) - self.log_debug("Current file info: %s" % self.info) + self.log_debug("File info: %s" % self.info) + self.log_debug("Previous file info: %s" % old_info) try: url = self.info['url'].strip() name = self.info['name'].strip() + + except KeyError: + pass + + else: if name and name is not url: self.pyfile.name = name - except Exception: - pass + if 'size' in self.info and self.info['size'] > 0: + self.pyfile.size = int(self.info['size']) #@TODO: Fix int conversion in 0.4.10 - try: - size = self.info['size'] - if size > 0: - self.pyfile.size = size + # self.pyfile.sync() - except Exception: - pass + name = self.pyfile.name + size = self.pyfile.size + folder = self.info['folder'] = name - self.log_debug("File name: %s" % self.pyfile.name, - "File size: %s byte" % self.pyfile.size if self.pyfile.size > 0 else "File size: Unknown") + self.log_info(_("File name: ") + name) + self.log_info(_("File size: %s bytes") % size if size > 0 else _("File size: Unknown")) + # self.log_info("File folder: " + folder) + #@TODO: Rewrite in 0.4.10 def check_info(self): self.check_name_size() @@ -507,13 +533,7 @@ class SimpleHoster(Hoster): def handle_direct(self, pyfile): - link = self.direct_link(pyfile.url, self.resume_download) - - if link: - self.log_info(_("Direct download link detected")) - self.link = link - else: - self.log_debug("Direct download link not found") + self.link = self.direct_link(pyfile.url, self.resume_download) def handle_multi(self, pyfile): #: Multi-hoster handler @@ -534,7 +554,7 @@ class SimpleHoster(Hoster): def handle_premium(self, pyfile): if not hasattr(self, 'LINK_PREMIUM_PATTERN'): self.log_error(_("Premium download not implemented")) - self.log_debug("Handled as free download") + self.log_info(_("Processing as free download...")) self.handle_free(pyfile) m = re.search(self.LINK_PREMIUM_PATTERN, self.html) @@ -542,12 +562,3 @@ class SimpleHoster(Hoster): self.error(_("Premium download link not found")) else: self.link = m.group(1) - - - def retry_free(self): - if not self.premium: - return - self.premium = False - self.account = None - self.req = self.pyload.requestFactory.getRequest(self.__name__) - raise Retry(_("Fallback to free download")) diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 6508b9bd3..962cffb06 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -13,7 +13,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.54" + __version__ = "0.55" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -27,7 +27,7 @@ class XFSHoster(SimpleHoster): HOSTER_DOMAIN = None - MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... + LEECH_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... NAME_PATTERN = r'(Filename[ ]*:[ ]*</b>(</td><td nowrap>)?|name="fname"[ ]+value="|<[\w^_]+ class="(file)?name">)\s*(?P<N>.+?)(\s*<|")' SIZE_PATTERN = r'(Size[ ]*:[ ]*</b>(</td><td>)?|File:.*>|</font>\s*\(|<[\w^_]+ class="size">)\s*(?P<S>[\d.,]+)\s*(?P<U>[\w^_]+)' @@ -110,8 +110,8 @@ class XFSHoster(SimpleHoster): if m: break else: - self.log_error(data['op'] if 'op' in data else _("UNKNOWN")) - return "" + if 'op' in data: + self.error(_("Missing OP data after: ") + data['op']) self.link = m.group(1) |