From ab3c90a7b0d18f47d3efc8259906895c1f50765d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 1 Oct 2015 23:21:14 +0200 Subject: Fixpack (1) --- module/plugins/hoster/TurbobitNet.py | 5 +- module/plugins/hoster/UploadingCom.py | 6 +-- module/plugins/internal/Base.py | 16 ++++-- module/plugins/internal/Crypter.py | 6 +-- module/plugins/internal/Hoster.py | 17 ++---- module/plugins/internal/Plugin.py | 19 ++++--- module/plugins/internal/SimpleCrypter.py | 41 ++++++++++++--- module/plugins/internal/SimpleHoster.py | 89 +++++++++++++++++++------------- module/plugins/internal/XFSCrypter.py | 1 + module/plugins/internal/XFSHoster.py | 6 +-- 10 files changed, 131 insertions(+), 75 deletions(-) (limited to 'module') diff --git a/module/plugins/hoster/TurbobitNet.py b/module/plugins/hoster/TurbobitNet.py index aad56788c..d20d1cadd 100644 --- a/module/plugins/hoster/TurbobitNet.py +++ b/module/plugins/hoster/TurbobitNet.py @@ -10,13 +10,14 @@ import urllib from Crypto.Cipher import ARC4 from module.plugins.captcha.ReCaptcha import ReCaptcha -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp +from module.plugins.internal.Plugin import timestamp +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class TurbobitNet(SimpleHoster): __name__ = "TurbobitNet" __type__ = "hoster" - __version__ = "0.21" + __version__ = "0.22" __status__ = "testing" __pattern__ = r'http://(?:www\.)?turbobit\.net/(?:download/free/)?(?P\w+)' diff --git a/module/plugins/hoster/UploadingCom.py b/module/plugins/hoster/UploadingCom.py index a7878f731..c8f8a8169 100644 --- a/module/plugins/hoster/UploadingCom.py +++ b/module/plugins/hoster/UploadingCom.py @@ -4,14 +4,14 @@ import pycurl import re from module.common.json_layer import json_loads -from module.plugins.internal.Plugin import encode -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, timestamp +from module.plugins.internal.Plugin import encode, timestamp +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class UploadingCom(SimpleHoster): __name__ = "UploadingCom" __type__ = "hoster" - __version__ = "0.43" + __version__ = "0.44" __status__ = "testing" __pattern__ = r'http://(?:www\.)?uploading\.com/files/(?:get/)?(?P\w+)' diff --git a/module/plugins/internal/Base.py b/module/plugins/internal/Base.py index 4235cf94d..843d2ad11 100644 --- a/module/plugins/internal/Base.py +++ b/module/plugins/internal/Base.py @@ -18,12 +18,22 @@ def getInfo(urls): pass +#@TODO: Remove in 0.4.10 +def parse_fileInfo(klass, url="", html=""): + info = klass.get_info(url, html) + return encode(info['name']), info['size'], info['status'], info['url'] + + #@TODO: Remove in 0.4.10 def create_getInfo(klass): def get_info(urls): for url in urls: - if hasattr(klass, "URL_REPLACEMENTS"): + try: url = replace_patterns(url, klass.URL_REPLACEMENTS) + + except Exception: + pass + yield parse_fileInfo(klass, url) return get_info @@ -42,7 +52,7 @@ def check_abort(fn): class Base(Plugin): __name__ = "Base" __type__ = "base" - __version__ = "0.01" + __version__ = "0.02" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -366,7 +376,7 @@ class Base(Plugin): @check_abort def load(self, *args, **kwargs): - return super(Hoster, self).load(*args, **kwargs) + return super(Base, self).load(*args, **kwargs) def check_abort(self): diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 069ffb589..a5c88aed9 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -7,7 +7,7 @@ from module.utils import save_path as safe_filename class Crypter(Base): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.10" + __version__ = "0.11" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -21,7 +21,7 @@ class Crypter(Base): def __init__(self, pyfile): - super(Base, self).__init__(pyfile) + super(Crypter, self).__init__(pyfile) #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) self.packages = [] @@ -34,7 +34,7 @@ class Crypter(Base): def _setup(self): - super(Base, self)._setup() + super(Crypter, self)._setup() self.packages = [] self.urls = [] diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index d1b894c6f..cb7b215ba 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -4,22 +4,15 @@ from __future__ import with_statement import os -from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo +from module.plugins.internal.Base import Base, check_abort, create_getInfo, getInfo, parse_fileInfo from module.plugins.internal.Plugin import Fail, Retry, encode, exists, fixurl, parse_name from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename -#@TODO: Remove in 0.4.10 -def parse_fileInfo(klass, url="", html=""): - info = klass.get_info(url, html) - return encode(info['name']), info['size'], info['status'], info['url'] - - - class Hoster(Base): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.32" + __version__ = "0.33" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -33,7 +26,7 @@ class Hoster(Base): def __init__(self, pyfile): - super(Base, self).__init__(pyfile) + super(Hoster, self).__init__(pyfile) #: Enable simultaneous processing of multiple downloads self.limitDL = 0 #@TODO: Change to `limit_dl` in 0.4.10 @@ -52,7 +45,7 @@ class Hoster(Base): def _setup(self): - super(Base, self)._setup() + super(Hoster, self)._setup() self.last_download = None self.last_check = None @@ -64,7 +57,7 @@ class Hoster(Base): self.account = False self.user = None #@TODO: Remove in 0.4.10 else: - super(Base, self).load_account() + super(Hoster, self).load_account() # self.rst_free = False diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index dbf6775cb..5725bfa58 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -6,6 +6,7 @@ import datetime import inspect import os import re +import time import traceback import urllib import urlparse @@ -76,6 +77,11 @@ def parse_name(string): return urllib.unquote(name) +#@TODO: Move to utils in 0.4.10 +def timestamp(): + return int(time.time() * 1000) + + #@TODO: Move to utils in 0.4.10 def which(program): """ @@ -145,7 +151,10 @@ def parse_html_form(attr_str, html, input_names={}): else: inputs[name] = value - if input_names: + if not input_names: + #: No attribute check + return action, inputs + else: #: Check input attributes for key, val in input_names.items(): if key in inputs: @@ -155,14 +164,12 @@ def parse_html_form(attr_str, html, input_names={}): continue elif hasattr(val, "search") and re.match(val, inputs[key]): continue - break #: Attibute value does not match + else: + break #: Attibute value does not match else: break #: Attibute name does not match else: return action, inputs #: Passed attribute check - else: - #: No attribute check - return action, inputs return {}, None #: No matching form found @@ -179,7 +186,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.40" + __version__ = "0.41" __status__ = "testing" __pattern__ = r'^unmatchable$' diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index d4ae85ebe..20e91ac1e 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -3,13 +3,14 @@ import re from module.plugins.internal.Crypter import Crypter -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookie, set_cookies +from module.plugins.internal.Plugin import replace_patterns, set_cookie, set_cookies +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.63" + __version__ = "0.64" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -47,13 +48,35 @@ class SimpleCrypter(Crypter, SimpleHoster): and its loadPage method: - def load_page(self, page_n): return the html of the page number page_n """ - DIRECT_LINK = True - LEECH_HOSTER = False + NAME_REPLACEMENTS = [] + URL_REPLACEMENTS = [] + + COOKIES = True #: or False or list of tuples [(domain, name, value)] + DIRECT_LINK = True #: Set to True to looking for direct link (as defined in handle_direct method), set to None to do it if self.account is True else False + LOGIN_ACCOUNT = False #: Set to True to require account login + LOGIN_PREMIUM = False #: Set to True to require premium account login + # LEECH_HOSTER = False #: Set to True to leech other hoster link (as defined in handle_multi method) + TEXT_ENCODING = True #: Set to encoding name if encoding value in http header is not correct + PAGES_PATTERN = None + + LINK_PATTERN = None + + NAME_PATTERN = None + HASHSUM_PATTERN = None + OFFLINE_PATTERN = None + TEMP_OFFLINE_PATTERN = None + + WAIT_PATTERN = None + PREMIUM_ONLY_PATTERN = None + HAPPY_HOUR_PATTERN = None + IP_BLOCKED_PATTERN = None + DL_LIMIT_PATTERN = None + SIZE_LIMIT_PATTERN = None + ERROR_PATTERN = None #@TODO: Remove in 0.4.10 @@ -113,7 +136,7 @@ class SimpleCrypter(Crypter, SimpleHoster): self.links = self.get_links() or list() - if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): + if self.PAGES_PATTERN: self.handle_pages(pyfile) self.log_debug("Package has %d links" % len(self.links)) @@ -134,11 +157,15 @@ class SimpleCrypter(Crypter, SimpleHoster): return re.findall(self.LINK_PATTERN, self.html) + def load_page(self, number) + raise NotImplementedError + + def handle_pages(self, pyfile): try: pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) - except (AttributeError, IndexError, ValueError): + except Exception: pages = 1 for p in xrange(2, pages + 1): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 8ba227c92..5d2307d51 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -16,7 +16,7 @@ from module.utils import fixup, fs_encode, parseFileSize as parse_size class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.89" + __version__ = "1.90" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -85,25 +85,43 @@ class SimpleHoster(Hoster): LINK_PREMIUM_PATTERN: (optional) group(1) should be the direct link for premium download example: LINK_PREMIUM_PATTERN = r'
)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)'), - ('Request error', r'([Aa]n error occured while processing your request)' ), - ('Html file' , r'\A\s*)?((?:[\w\s]*(?:[Ee]rror|ERROR)\s*\:?)?\s*\d{3})(?:\Z|\s+)'), + ('Request error', r'([Aa]n error occured while processing your request)' ), + ('Html file' , r'\A\s* 300 self.retry(1, wait_time, _("Download limit exceeded")) - if hasattr(self, 'HAPPY_HOUR_PATTERN') and re.search(self.HAPPY_HOUR_PATTERN, self.html): + if self.HAPPY_HOUR_PATTERN and re.search(self.HAPPY_HOUR_PATTERN, self.html): self.multiDL = True - if hasattr(self, 'ERROR_PATTERN'): + if self.ERROR_PATTERN: m = re.search(self.ERROR_PATTERN, self.html) if m is not None: try: @@ -396,7 +414,7 @@ class SimpleHoster(Hoster): self.wantReconnect = True self.retry(wait=60, msg=errmsg) - elif hasattr(self, 'WAIT_PATTERN'): + elif self.WAIT_PATTERN: m = re.search(self.WAIT_PATTERN, self.html) if m is not None: try: @@ -496,7 +514,7 @@ class SimpleHoster(Hoster): def handle_free(self, pyfile): - if not hasattr(self, 'LINK_FREE_PATTERN'): + if not self.LINK_FREE_PATTERN: self.log_error(_("Free download not implemented")) m = re.search(self.LINK_FREE_PATTERN, self.html) @@ -507,10 +525,9 @@ class SimpleHoster(Hoster): def handle_premium(self, pyfile): - if not hasattr(self, 'LINK_PREMIUM_PATTERN'): + if not self.LINK_PREMIUM_PATTERN: self.log_error(_("Premium download not implemented")) - self.log_info(_("Processing as free download...")) - self.handle_free(pyfile) + self.restart() m = re.search(self.LINK_PREMIUM_PATTERN, self.html) if m is None: diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 69565e22c..876575874 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -30,6 +30,7 @@ class XFSCrypter(SimpleCrypter): OFFLINE_PATTERN = r'>\s*(No such user|\w+ (Not Found|file (was|has been) removed|no longer available)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' + def set_xfs_cookie(self): if not self.COOKIES: return diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index dcb85d23b..b59f34122 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.62" + __version__ = "0.63" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -253,7 +253,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) - except (AttributeError, IndexError, TypeError): + except Exception: captcha_key = recaptcha.detect_key() else: @@ -267,7 +267,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) - except (AttributeError, IndexError, TypeError): + except Exception: captcha_key = solvemedia.detect_key() else: -- cgit v1.2.3