diff options
Diffstat (limited to 'pyload/plugins/internal')
| -rw-r--r-- | pyload/plugins/internal/BasePlugin.py | 90 | ||||
| -rw-r--r-- | pyload/plugins/internal/DeadCrypter.py | 16 | ||||
| -rw-r--r-- | pyload/plugins/internal/DeadHoster.py | 24 | ||||
| -rw-r--r-- | pyload/plugins/internal/SimpleCrypter.py | 4 | ||||
| -rw-r--r-- | pyload/plugins/internal/SimpleHoster.py | 39 | ||||
| -rw-r--r-- | pyload/plugins/internal/XFSAccount.py | 51 | ||||
| -rw-r--r-- | pyload/plugins/internal/XFSHoster.py | 37 |
7 files changed, 157 insertions, 104 deletions
diff --git a/pyload/plugins/internal/BasePlugin.py b/pyload/plugins/internal/BasePlugin.py index dd8540578..f4abc1a15 100644 --- a/pyload/plugins/internal/BasePlugin.py +++ b/pyload/plugins/internal/BasePlugin.py @@ -6,20 +6,26 @@ from urllib import unquote from urlparse import urlparse from pyload.network.HTTPRequest import BadHeader +from pyload.plugins.internal.SimpleHoster import create_getInfo from pyload.plugins.internal.Hoster import Hoster -from pyload.utils import html_unescape, remove_chars class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" - __version__ = "0.20" + __version__ = "0.23" __pattern__ = r'^unmatchable$' __description__ = """Base Plugin when any other didnt fit""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org")] + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + @classmethod + def getInfo(cls, url="", html=""): #@TODO: Move to hoster class in 0.4.10 + return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url or ""} def setup(self): @@ -30,40 +36,45 @@ class BasePlugin(Hoster): def process(self, pyfile): """main function""" - #: debug part, for api exerciser - if pyfile.url.startswith("DEBUG_API"): - self.multiDL = False - return + pyfile.name = self.getInfo(pyfile.url)['name'] if pyfile.url.startswith("http"): + for _i in xrange(2): + try: + self.downloadFile(pyfile) - try: - self.downloadFile(pyfile) - except BadHeader, e: - if e.code in (401, 403): - self.logDebug("Auth required") + except BadHeader, e: + if e.code is 404: + self.offline() - account = self.core.accountManager.getAccountPlugin('Http') - servers = [x['login'] for x in account.getAllAccounts()] - server = urlparse(pyfile.url).netloc + elif e.code in (401, 403): + self.logDebug("Auth required") - if server in servers: - self.logDebug("Logging on to %s" % server) - self.req.addAuth(account.accounts[server]['password']) - else: - for pwd in pyfile.package().password.splitlines(): - if ":" in pwd: - self.req.addAuth(pwd.strip()) - break - else: - self.fail(_("Authorization required (username:password)")) + account = self.core.accountManager.getAccountPlugin('Http') + servers = [x['login'] for x in account.getAllAccounts()] + server = urlparse(pyfile.url).netloc - self.downloadFile(pyfile) + if server in servers: + self.logDebug("Logging on to %s" % server) + self.req.addAuth(account.accounts[server]['password']) + else: + for pwd in pyfile.package().password.splitlines(): + if ":" in pwd: + self.req.addAuth(pwd.strip()) + break + else: + self.fail(_("Authorization required (username:password)")) + else: + self.fail(e) else: - raise - + break + else: + self.fail(_("No file downloaded")) #@TODO: Move to hoster class (check if self.lastDownload) in 0.4.10 else: - self.fail(_("No Plugin matched and not a downloadable url")) + self.fail(_("No plugin matched")) + + # if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": + # self.fail(_("Empty file")) def downloadFile(self, pyfile): @@ -78,31 +89,18 @@ class BasePlugin(Hoster): if 'location' in header: self.logDebug("Location: " + header['location']) + base = re.match(r'https?://[^/]+', url).group(0) + if header['location'].startswith("http"): url = header['location'] + elif header['location'].startswith("/"): url = base + unquote(header['location']) + else: url = '%s/%s' % (base, unquote(header['location'])) else: break - name = html_unescape(unquote(urlparse(url).path.split("/")[-1])) - - if 'content-disposition' in header: - self.logDebug("Content-Disposition: " + header['content-disposition']) - m = re.search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", header['content-disposition']) - if m: - disp = m.groupdict() - self.logDebug(disp) - if not disp['enc']: - disp['enc'] = 'utf-8' - name = remove_chars(disp['name'], "\"';").strip() - name = unicode(unquote(name), disp['enc']) - - if not name: - name = url - pyfile.name = name - self.logDebug("Filename: %s" % pyfile.name) self.download(url, disposition=True) diff --git a/pyload/plugins/internal/DeadCrypter.py b/pyload/plugins/internal/DeadCrypter.py index bf150f3d5..3510e1466 100644 --- a/pyload/plugins/internal/DeadCrypter.py +++ b/pyload/plugins/internal/DeadCrypter.py @@ -1,12 +1,15 @@ # -*- coding: utf-8 -*- +from urlparse import urlparse + from pyload.plugins.internal.Crypter import Crypter as _Crypter +from pyload.plugins.internal.SimpleCrypter import create_getInfo class DeadCrypter(_Crypter): __name__ = "DeadCrypter" __type__ = "crypter" - __version__ = "0.02" + __version__ = "0.03" __pattern__ = r'^unmatchable$' @@ -15,5 +18,14 @@ class DeadCrypter(_Crypter): __authors__ = [("stickell", "l.stickell@yahoo.it")] + @classmethod + def getInfo(cls, url="", html=""): + return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 1, 'url': url or ""} + + def setup(self): - self.offline("Crypter is no longer available") + self.pyfile.error = "Crypter is no longer available" + self.offline() #@TODO: self.offline("Crypter is no longer available") + + +getInfo = create_getInfo(DeadCrypter) diff --git a/pyload/plugins/internal/DeadHoster.py b/pyload/plugins/internal/DeadHoster.py index 036ed3cb6..a7e5093d3 100644 --- a/pyload/plugins/internal/DeadHoster.py +++ b/pyload/plugins/internal/DeadHoster.py @@ -1,20 +1,15 @@ # -*- coding: utf-8 -*- -from pyload.plugins.internal.Hoster import Hoster as _Hoster - - -def create_getInfo(plugin): +from urlparse import urlparse - def getInfo(urls): - yield map(lambda url: ('#N/A: ' + url, 0, 1, url), urls) - - return getInfo +from pyload.plugins.internal.Hoster import Hoster as _Hoster +from pyload.plugins.internal.SimpleHoster import create_getInfo class DeadHoster(_Hoster): __name__ = "DeadHoster" __type__ = "hoster" - __version__ = "0.12" + __version__ = "0.13" __pattern__ = r'^unmatchable$' @@ -23,5 +18,14 @@ class DeadHoster(_Hoster): __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + @classmethod + def getInfo(cls, url="", html=""): + return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 1, 'url': url or ""} + + def setup(self): - self.offline("Hoster is no longer available") + self.pyfile.error = "Hoster is no longer available" + self.offline() #@TODO: self.offline("Hoster is no longer available") + + +getInfo = create_getInfo(DeadHoster) diff --git a/pyload/plugins/internal/SimpleCrypter.py b/pyload/plugins/internal/SimpleCrypter.py index ead5cefba..090cf2e9f 100644 --- a/pyload/plugins/internal/SimpleCrypter.py +++ b/pyload/plugins/internal/SimpleCrypter.py @@ -5,14 +5,14 @@ import re from urlparse import urlparse from pyload.plugins.internal.Crypter import Crypter -from pyload.plugins.internal.SimpleHoster import SimpleHoster, replace_patterns, set_cookies +from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies from pyload.utils import fixup class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.31" + __version__ = "0.32" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] diff --git a/pyload/plugins/internal/SimpleHoster.py b/pyload/plugins/internal/SimpleHoster.py index 4e9db7f73..a33e48bdf 100644 --- a/pyload/plugins/internal/SimpleHoster.py +++ b/pyload/plugins/internal/SimpleHoster.py @@ -37,12 +37,13 @@ def parseHtmlTagAttrValue(attr_name, tag): return m.group(2) if m else None -def parseHtmlForm(attr_str, html, input_names=None): - for form in re.finditer(r"(?P<tag><form[^>]*%s[^>]*>)(?P<content>.*?)</?(form|body|html)[^>]*>" % attr_str, +def parseHtmlForm(attr_str, html, input_names={}): + for form in re.finditer(r"(?P<TAG><form[^>]*%s[^>]*>)(?P<CONTENT>.*?)</?(form|body|html)[^>]*>" % attr_str, html, re.S | re.I): inputs = {} - action = parseHtmlTagAttrValue("action", form.group('tag')) - for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('content'), re.S | re.I): + action = parseHtmlTagAttrValue("action", form.group('TAG')) + + for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.S | re.I): name = parseHtmlTagAttrValue("name", inputtag.group(1)) if name: value = parseHtmlTagAttrValue("value", inputtag.group(1)) @@ -51,7 +52,7 @@ def parseHtmlForm(attr_str, html, input_names=None): else: inputs[name] = value - if isinstance(input_names, dict): + if input_names: # check input attributes for key, val in input_names.iteritems(): if key in inputs: @@ -80,6 +81,7 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 +#@NOTE: Every plugin must have own parseInfo classmethod to work with 0.4.10 def create_getInfo(plugin): return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfo(urls)] @@ -90,23 +92,21 @@ def timestamp(): #@TODO: Move to hoster class in 0.4.10 def _getDirectLink(self, url): - self.req.http.c.setopt(FOLLOWLOCATION, 0) + header = self.load(url, ref=True, just_header=True, decode=True) - html = self.load(url, ref=True, decode=True) + if not 'location' in header or not header['location']: + return "" - self.req.http.c.setopt(FOLLOWLOCATION, 1) + if header['code'] != 302 or 'content-type' not in header or header['content-type'] != "text/plain": + return "" - if self.getInfo(url, html)['status'] is not 2: - try: - return re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I).group(1).rstrip() #@TODO: Remove .rstrip() in 0.4.10 - except: - pass + return header['location'] class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.60" + __version__ = "0.62" __pattern__ = r'^unmatchable$' @@ -177,7 +177,7 @@ class SimpleHoster(Hoster): @classmethod def getInfo(cls, url="", html=""): - info = {'name': url or _("Unknown"), 'size': 0, 'status': 3, 'url': url} + info = {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url or ""} if not html: if url: @@ -391,14 +391,15 @@ class SimpleHoster(Hoster): def handleDirect(self): - self.link = _getDirectLink(self, self.pyfile.url) + link = _getDirectLink(self, self.pyfile.url) - if self.link: + if link: self.logInfo(_("Direct download link detected")) + self.link = link + self._updateInfo(self.getInfo(self.pyfile.url)) self.checkNameSize() - else: self.logDebug(_("Direct download link not found")) @@ -452,7 +453,7 @@ class SimpleHoster(Hoster): self.retry(max_tries=max_tries, reason=_("Download limit reached")) - def parseHtmlForm(self, attr_str='', input_names=None): + def parseHtmlForm(self, attr_str="", input_names={}): return parseHtmlForm(attr_str, self.html, input_names) diff --git a/pyload/plugins/internal/XFSAccount.py b/pyload/plugins/internal/XFSAccount.py index 1e18c09bd..168c4f903 100644 --- a/pyload/plugins/internal/XFSAccount.py +++ b/pyload/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from pyload.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.26" + __version__ = "0.30" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -32,6 +32,9 @@ class XFSAccount(Account): TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?<b>\s*(?P<S>[\d.,]+|[Uu]nlimited)\s*(?:(?P<U>[\w^_]+)\s*)?</b>' TRAFFIC_LEFT_UNIT = "MB" #: used only if no group <U> was found + LEECH_TRAFFIC_PATTERN = r'Leech Traffic left:<b>.*?(?P<S>[\d.,]+|[Uu]nlimited)\s*(?:(?P<U>[\w^_]+)\s*)?</b>' + LEECH_TRAFFIC_UNIT = "MB" #: used only if no group <U> was found + LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' @@ -44,9 +47,10 @@ class XFSAccount(Account): def loadAccountInfo(self, user, req): - validuntil = None - trafficleft = None - premium = None + validuntil = None + trafficleft = None + leechtraffic = None + premium = None html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) @@ -64,17 +68,22 @@ class XFSAccount(Account): self.logError(e) else: + self.logDebug("Valid until: %s" % validuntil) + if validuntil > mktime(gmtime()): premium = True + trafficleft = -1 else: premium = False validuntil = None #: registered account type (not premium) + else: + self.logDebug("VALID_UNTIL_PATTERN not found") m = re.search(self.TRAFFIC_LEFT_PATTERN, html) if m: try: traffic = m.groupdict() - size = traffic['S'] + size = traffic['S'] if "nlimited" in size: trafficleft = -1 @@ -93,10 +102,36 @@ class XFSAccount(Account): except Exception, e: self.logError(e) else: - if premium: - trafficleft = -1 + self.logDebug("TRAFFIC_LEFT_PATTERN not found") + + m = re.finditer(self.LEECH_TRAFFIC_PATTERN, html) + if m: + leechtraffic = 0 + try: + for leech in m: + size = leech['S'] + + if "nlimited" in size: + leechtraffic = -1 + if validuntil is None: + validuntil = -1 + break + else: + if 'U' in leech: + unit = leech['U'] + elif isinstance(self.LEECH_TRAFFIC_UNIT, basestring): + unit = self.LEECH_TRAFFIC_UNIT + else: + unit = "" + + leechtraffic += self.parseTraffic(size + unit) + + except Exception, e: + self.logError(e) + else: + self.logDebug("LEECH_TRAFFIC_PATTERN not found") - return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} + return {'validuntil': validuntil, 'trafficleft': trafficleft, 'leechtraffic': leechtraffic, 'premium': premium} def login(self, user, data, req): diff --git a/pyload/plugins/internal/XFSHoster.py b/pyload/plugins/internal/XFSHoster.py index 3ae9ee05a..4d3c848b9 100644 --- a/pyload/plugins/internal/XFSHoster.py +++ b/pyload/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from pyload.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.22" + __version__ = "0.26" __pattern__ = r'^unmatchable$' @@ -30,15 +30,13 @@ class XFSHoster(SimpleHoster): HOSTER_DOMAIN = None HOSTER_NAME = None - URL_REPLACEMENTS = [(r'/(?:embed-)?(\w{12}).*', r'/\1')] #: plus support embedded files - TEXT_ENCODING = False COOKIES = [(HOSTER_DOMAIN, "lang", "english")] CHECK_DIRECT_LINK = None - MULTI_HOSTER = False + MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... INFO_PATTERN = r'<tr><td align=right><b>Filename:</b></td><td nowrap>(?P<N>[^<]+)</td></tr>\s*.*?<small>\((?P<S>[^<]+)\)</small>' - NAME_PATTERN = r'(>Filename:</b></td><td nowrap>|name="fname" value="|<span class="name">|<[Tt]itle>.*?Download )(?P<N>.+?)(\s*<|")' + NAME_PATTERN = r'(>Filename:</b></td><td nowrap>|name="fname" value="|<span class="name">)(?P<N>.+?)(\s*<|")' SIZE_PATTERN = r'(>Size:</b></td><td>|>File:.*>|<span class="size">)(?P<S>[\d.,]+)\s*(?P<U>[\w^_]+)' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' @@ -48,14 +46,17 @@ class XFSHoster(SimpleHoster): PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only' ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter><b>|>Error</td>|>\(ERROR:)(?:\s*<.+?>\s*)*(.+?)(?:["\']|<|\))' - OVR_LINK_PATTERN = r'<h2>Download Link</h2>\s*<textarea[^>]*>([^<]+)' - LINK_PATTERN = None #: final download url pattern + LEECH_LINK_PATTERN = r'<h2>Download Link</h2>\s*<textarea[^>]*>([^<]+)' + LINK_PATTERN = None #: final download url pattern CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' CAPTCHA_DIV_PATTERN = r'>Enter code.*?<div.*?>(.+?)</div>' RECAPTCHA_PATTERN = None SOLVEMEDIA_PATTERN = None + FORM_PATTERN = None + FORM_INPUTS_MAP = None #: dict passed as input_names to parseHtmlForm + def setup(self): self.chunkLimit = 1 @@ -108,7 +109,7 @@ class XFSHoster(SimpleHoster): def getDownloadLink(self): - for i in xrange(1, 5): + for i in xrange(1, 6): self.logDebug("Getting download link: #%d" % i) self.checkErrors() @@ -145,7 +146,7 @@ class XFSHoster(SimpleHoster): #only tested with easybytez.com self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) - action, inputs = self.parseHtmlForm('') + action, inputs = self.parseHtmlForm() upload_id = "%012d" % int(random() * 10 ** 12) action += upload_id + "&js_on=1&utype=prem&upload_type=url" @@ -186,9 +187,9 @@ class XFSHoster(SimpleHoster): self.fail(stmsg) #get easybytez.com link for uploaded file - m = re.search(self.OVR_LINK_PATTERN, self.html) + m = re.search(self.LEECH_LINK_PATTERN, self.html) if m is None: - self.error(_("OVR_LINK_PATTERN not found")) + self.error(_("LEECH_LINK_PATTERN not found")) header = self.load(m.group(1), just_header=True, decode=True) @@ -259,10 +260,10 @@ class XFSHoster(SimpleHoster): def getPostParameters(self): - if hasattr(self, "FORM_PATTERN"): - action, inputs = self.parseHtmlForm(self.FORM_PATTERN) + if self.FORM_PATTERN or self.FORM_INPUTS_MAP: + action, inputs = self.parseHtmlForm(self.FORM_PATTERN or "", self.FORM_INPUTS_MAP or {}) else: - action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) + action, inputs = self.parseHtmlForm(input_names={'op': re.compile(r'^download')}) if not inputs: action, inputs = self.parseHtmlForm('F1') @@ -313,8 +314,8 @@ class XFSHoster(SimpleHoster): m = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.S) if m: captcha_div = m.group(1) + numerals = re.findall(r'<span.*?padding-left\s*:\s*(\d+).*?>(\d)</span>', html_unescape(captcha_div)) self.logDebug(captcha_div) - numerals = re.findall(r'<span.*?padding-left\s*:\s*(\d+).*?>(\d)</span>', html_unescape(captcha_div)) inputs['code'] = "".join([a[1] for a in sorted(numerals, key=lambda num: int(num[0]))]) self.logDebug("Captcha code: %s" % inputs['code'], numerals) return 2 @@ -324,9 +325,10 @@ class XFSHoster(SimpleHoster): captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) except: captcha_key = recaptcha.detect_key() + else: + self.logDebug("ReCaptcha key: %s" % captcha_key) if captcha_key: - self.logDebug("ReCaptcha key: %s" % captcha_key) inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) return 3 @@ -335,9 +337,10 @@ class XFSHoster(SimpleHoster): captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) except: captcha_key = solvemedia.detect_key() + else: + self.logDebug("SolveMedia key: %s" % captcha_key) if captcha_key: - self.logDebug("SolveMedia key: %s" % captcha_key) inputs['adcopy_challenge'], inputs['adcopy_response'] = solvemedia.challenge(captcha_key) return 4 |