From 2d893a24d4bb8b5a7a00807ed6320358454577f8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 29 Jun 2014 21:55:29 +0200 Subject: [SimpleHoster] Improve documentation of own public attributes --- module/plugins/internal/SimpleHoster.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index f10433e78..8c2e3daae 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -169,15 +169,19 @@ class SimpleHoster(Hoster): __description__ = """Simple hoster plugin""" __author_name__ = ("zoidberg", "stickell") __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") + """ - These patterns should be defined by each hoster: - FILE_INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Punits)' - or FILE_NAME_PATTERN = r'(?Pfile_name)' - and FILE_SIZE_PATTERN = r'(?Pfile_size) (?Punits)' + Following patterns should be defined by each hoster: + FILE_INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' + or + FILE_NAME_PATTERN = r'(?Pfile_name)' + FILE_SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' + FILE_OFFLINE_PATTERN = r'File (deleted|not found)' TEMP_OFFLINE_PATTERN = r'Server maintainance' - You can also define a PREMIUM_ONLY_PATTERN to detect links that can be downloaded only with a premium account. + You can also define: + PREMIUM_ONLY_PATTERN = r'Premium account required' """ FILE_SIZE_REPLACEMENTS = [] -- cgit v1.2.3 From 9c9cc6d4871b9847dada36c63ae1cbadfacef999 Mon Sep 17 00:00:00 2001 From: Stefano Date: Wed, 2 Jul 2014 15:23:46 +0200 Subject: [SimpleCrypter] ability to set cookies --- module/plugins/internal/SimpleCrypter.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index b1a18f5e0..268efee86 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -21,12 +21,12 @@ import re from module.plugins.Crypter import Crypter from module.utils import html_unescape -from module.plugins.internal.SimpleHoster import replace_patterns +from module.plugins.internal.SimpleHoster import replace_patterns, set_cookies class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" - __version__ = "0.07" + __version__ = "0.08" __pattern__ = None __type__ = "crypter" __description__ = """Simple decrypter plugin""" @@ -55,6 +55,12 @@ class SimpleCrypter(Crypter): FILE_URL_REPLACEMENTS = [] + SH_COOKIES = True # or False or list of tuples [(domain, name, value)] + + def setup(self): + if isinstance(self.SH_COOKIES, list): + set_cookies(self.req.cj, self.SH_COOKIES) + def decrypt(self, pyfile): pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS) -- cgit v1.2.3 From 15362ebceaee1eb66920d4822d2faa947151e74a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 5 Jul 2014 16:42:24 +0200 Subject: [SimpleCrypter] Better inline docs + support for offline & temp.offline check --- module/plugins/internal/SimpleCrypter.py | 40 +++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 11 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 268efee86..2c024d738 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -26,34 +26,44 @@ from module.plugins.internal.SimpleHoster import replace_patterns, set_cookies class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" - __version__ = "0.08" + __version__ = "0.09" __pattern__ = None __type__ = "crypter" __description__ = """Simple decrypter plugin""" __author_name__ = ("stickell", "zoidberg") __author_mail__ = ("l.stickell@yahoo.it", "zoidberg@mujmail.cz") + """ - These patterns should be defined by each crypter: + Following patterns should be defined by each crypter: + + LINK_PATTERN: group(1) must be a download link or a regex to catch more links + example: LINK_PATTERN = r'
Date: Sat, 5 Jul 2014 16:54:20 +0200 Subject: [SimpleHoster] Better inline docs + changed "FILE_OFFLINE_PATTERN" to "OFFLINE_PATTERN" --- module/plugins/internal/SimpleHoster.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 8c2e3daae..b5066437e 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -98,7 +98,8 @@ def parseFileInfo(self, url='', html=''): if hasattr(self, "html"): self.html = html - if hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html): + if (hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html)) or \ + (hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html)): # File offline info['status'] = 1 else: @@ -163,7 +164,7 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" - __version__ = "0.33" + __version__ = "0.34" __pattern__ = None __type__ = "hoster" __description__ = """Simple hoster plugin""" @@ -172,16 +173,25 @@ class SimpleHoster(Hoster): """ Following patterns should be defined by each hoster: - FILE_INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' - or - FILE_NAME_PATTERN = r'(?Pfile_name)' - FILE_SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' - FILE_OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN = r'Server maintainance' + FILE_INFO_PATTERN: Name and Size of the file + example: FILE_INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' + or + FILE_NAME_PATTERN: Name that will be set for the file + example: FILE_NAME_PATTERN = r'(?Pfile_name)' + FILE_SIZE_PATTERN: Size that will be checked for the file + example: FILE_SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' - You can also define: - PREMIUM_ONLY_PATTERN = r'Premium account required' + OFFLINE_PATTERN: Checks if the file is yet available online + example: OFFLINE_PATTERN = r'File (deleted|not found)' + or: + FILE_OFFLINE_PATTERN: Deprecated + + TEMP_OFFLINE_PATTERN: Checks if the file is temporarily offline + example: TEMP_OFFLINE_PATTERN = r'Server maintainance' + + PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account + example: PREMIUM_ONLY_PATTERN = r'Premium account required' """ FILE_SIZE_REPLACEMENTS = [] -- cgit v1.2.3 From 0ba3f71ba9bf5c500245ca40ac2eceb6a31da52b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 5 Jul 2014 20:46:08 +0200 Subject: [SimpleCrypter] Added missed PAGES_PATTERN example --- module/plugins/internal/SimpleCrypter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 2c024d738..5cf3b55eb 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -54,7 +54,7 @@ class SimpleCrypter(Crypter): If the links are disposed on multiple pages you need to define a pattern: PAGES_PATTERN: The group defined by 'pages' must be the total number of pages - example: PAGES_PATTERN = r'' + example: PAGES_PATTERN = r'Pages: (?P\d+)' and a function: -- cgit v1.2.3 From 632139090a0440d2e61631a5ef4e3eafd728f9c7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 10 Jul 2014 02:03:02 +0200 Subject: [SimpleCrypter] Support parseError --- module/plugins/internal/SimpleCrypter.py | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 5cf3b55eb..2bcdad6a3 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -1,37 +1,20 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: zoidberg -""" - import re from module.plugins.Crypter import Crypter from module.utils import html_unescape -from module.plugins.internal.SimpleHoster import replace_patterns, set_cookies +from module.plugins.internal.SimpleHoster import PluginParseError, replace_patterns, set_cookies class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" - __version__ = "0.09" + __version__ = "0.10" __pattern__ = None __type__ = "crypter" __description__ = """Simple decrypter plugin""" - __author_name__ = ("stickell", "zoidberg") - __author_mail__ = ("l.stickell@yahoo.it", "zoidberg@mujmail.cz") + __author_name__ = ("stickell", "zoidberg", "Walter Purcaro") + __author_mail__ = ("l.stickell@yahoo.it", "zoidberg@mujmail.cz", "vuolter@gmail.com") """ Following patterns should be defined by each crypter: @@ -128,3 +111,6 @@ class SimpleCrypter(Crypter): for p in xrange(2, pages + 1): self.html = self.loadPage(p) self.package_links += self.getLinks() + + def parseError(self, msg): + raise PluginParseError(msg) -- cgit v1.2.3 From 04038a2cf0c4c2d9cc9a0c8e8bf9beb6426afae8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 10 Jul 2014 03:02:26 +0200 Subject: Use parseError instead PluginParseError + unified all download pattern attributes as LINK_PATTERN + removed some old patterns (not used anymore) + other code cosmetics --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index b5066437e..8e6b8b42f 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -194,8 +194,8 @@ class SimpleHoster(Hoster): example: PREMIUM_ONLY_PATTERN = r'Premium account required' """ - FILE_SIZE_REPLACEMENTS = [] FILE_NAME_REPLACEMENTS = [("&#?\w+;", fixup)] + FILE_SIZE_REPLACEMENTS = [] FILE_URL_REPLACEMENTS = [] SH_BROKEN_ENCODING = False # Set to True or encoding name if encoding in http header is not correct -- cgit v1.2.3 From 08ee77db1b3ec0cd42e40e35c07c6c3882d940bf Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 10 Jul 2014 03:31:21 +0200 Subject: [CaptchaService] Remove SolveMedia __init__ --- module/plugins/internal/CaptchaService.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 400484d26..30f9e8d14 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -1,28 +1,11 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: zoidberg -""" - import re from random import random class CaptchaService(): - __version__ = "0.04" + __version__ = "0.05" def __init__(self, plugin): self.plugin = plugin @@ -88,9 +71,6 @@ class AdsCaptcha(CaptchaService): class SolveMedia(CaptchaService): - def __init__(self, plugin): - self.plugin = plugin - def challenge(self, src): html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript?k=%s" % src, cookies=True) try: -- cgit v1.2.3 From 48c0c42fd6faffc56432d5f037cd575979f180cc Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 14 Jul 2014 02:23:37 +0200 Subject: Removed all @author flags + key attributes cleanup for internal & hooks plugins --- module/plugins/internal/AbstractExtractor.py | 7 ++++++- module/plugins/internal/CaptchaService.py | 7 +++++++ module/plugins/internal/DeadCrypter.py | 5 ++++- module/plugins/internal/DeadHoster.py | 5 ++++- module/plugins/internal/MultiHoster.py | 10 ++++++---- module/plugins/internal/SimpleCrypter.py | 6 ++++-- module/plugins/internal/SimpleHoster.py | 7 ++++--- module/plugins/internal/UnRar.py | 7 +++++-- module/plugins/internal/UnZip.py | 7 +++++-- module/plugins/internal/XFSPAccount.py | 4 ++-- 10 files changed, 47 insertions(+), 18 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index 0ecc11f06..c7f54f847 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -14,9 +14,14 @@ class WrongPassword(Exception): class AbtractExtractor: - + __name__ = "AbtractExtractor" __version__ = "0.1" + __description__ = """Abtract extractor plugin""" + __author_name__ = "pyLoad Team" + __author_mail__ = "admin@pyload.org" + + @staticmethod def checkDeps(): """ Check if system statisfy dependencies diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 30f9e8d14..c2c66ed31 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -5,8 +5,14 @@ from random import random class CaptchaService(): + __name__ = "CaptchaService" __version__ = "0.05" + __description__ = """Captcha service plugin""" + __author_name__ = "pyLoad Team" + __author_mail__ = "admin@pyload.org" + + def __init__(self, plugin): self.plugin = plugin @@ -17,6 +23,7 @@ class ReCaptcha(): recaptcha_key = None + def __init__(self, plugin): self.plugin = plugin diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 10eccb9bd..d19731cb5 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -5,12 +5,15 @@ from module.plugins.Crypter import Crypter as _Crypter class DeadCrypter(_Crypter): __name__ = "DeadCrypter" + __version__ = "0.01" __type__ = "crypter" + __pattern__ = None - __version__ = "0.01" + __description__ = """Crypter is no longer available""" __author_name__ = "stickell" __author_mail__ = "l.stickell@yahoo.it" + def setup(self): self.fail("Crypter is no longer available") diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index 201835e2b..77ee7c74f 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -11,12 +11,15 @@ def create_getInfo(plugin): class DeadHoster(_Hoster): __name__ = "DeadHoster" + __version__ = "0.11" __type__ = "hoster" + __pattern__ = None - __version__ = "0.11" + __description__ = """Hoster is no longer available""" __author_name__ = "zoidberg" __author_mail__ = "zoidberg@mujmail.cz" + def setup(self): self.fail("Hoster is no longer available") diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 97f3a5996..0dea76109 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -7,12 +7,13 @@ from module.plugins.Hook import Hook class MultiHoster(Hook): - """ - Generic MultiHoster plugin - """ - + __name__ = "AbtractExtractor" __version__ = "0.19" + __description__ = """Generic MultiHoster plugin""" + __author_name__ = "pyLoad Team" + __author_mail__ = "admin@pyload.org" + replacements = [("2shared.com", "twoshared.com"), ("4shared.com", "fourshared.com"), ("cloudnator.com", "shragle.com"), ("ifile.it", "filecloud.io"), ("easy-share.com", "crocko.com"), ("freakshare.net", "freakshare.com"), ("hellshare.com", "hellshare.cz"), ("share-rapid.cz", "sharerapid.com"), ("sharerapid.cz", "sharerapid.com"), @@ -20,6 +21,7 @@ class MultiHoster(Hook): ignored = [] interval = 24 * 60 * 60 #: reload hosters daily + def setup(self): self.hosters = [] self.supported = [] diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 2bcdad6a3..dc93bef4d 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,8 +10,10 @@ from module.plugins.internal.SimpleHoster import PluginParseError, replace_patte class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __version__ = "0.10" - __pattern__ = None __type__ = "crypter" + + __pattern__ = None + __description__ = """Simple decrypter plugin""" __author_name__ = ("stickell", "zoidberg", "Walter Purcaro") __author_mail__ = ("l.stickell@yahoo.it", "zoidberg@mujmail.cz", "vuolter@gmail.com") @@ -47,9 +49,9 @@ class SimpleCrypter(Crypter): URL_REPLACEMENTS = [] - SH_COOKIES = True # or False or list of tuples [(domain, name, value)] + def setup(self): if isinstance(self.SH_COOKIES, list): set_cookies(self.req.cj, self.SH_COOKIES) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 8e6b8b42f..e9ba59c0d 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -13,8 +13,6 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - - @author: zoidberg """ from urlparse import urlparse import re @@ -165,8 +163,10 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __version__ = "0.34" - __pattern__ = None __type__ = "hoster" + + __pattern__ = None + __description__ = """Simple hoster plugin""" __author_name__ = ("zoidberg", "stickell") __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") @@ -202,6 +202,7 @@ class SimpleHoster(Hoster): SH_COOKIES = True # or False or list of tuples [(domain, name, value)] SH_CHECK_TRAFFIC = False # True = force check traffic left for a premium account + def init(self): self.file_info = {} diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index e3765602b..28f801f3e 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -13,8 +13,6 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - - @author: RaNaN """ import os @@ -32,6 +30,10 @@ class UnRar(AbtractExtractor): __name__ = "UnRar" __version__ = "0.16" + __description__ = """Rar extractor plugin""" + __author_name__ = "RaNaN" + __author_mail__ = "RaNaN@pyload.org" + # there are some more uncovered rar formats re_version = re.compile(r"(UNRAR 5[\.\d]+(.*?)freeware)") re_splitfile = re.compile(r"(.*)\.part(\d+)\.rar$", re.I) @@ -41,6 +43,7 @@ class UnRar(AbtractExtractor): re_wrongpwd = re.compile("(Corrupt file or wrong password|password incorrect)", re.I) CMD = "unrar" + @staticmethod def checkDeps(): if os.name == "nt": diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 501962442..bace76789 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -13,8 +13,6 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - - @author: RaNaN """ import zipfile @@ -27,6 +25,11 @@ class UnZip(AbtractExtractor): __name__ = "UnZip" __version__ = "0.1" + __description__ = """Zip extractor plugin""" + __author_name__ = "RaNaN" + __author_mail__ = "RaNaN@pyload.org" + + @staticmethod def checkDeps(): return sys.version_info[:2] >= (2, 6) diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index 76aff54f0..b58eaf15c 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -13,8 +13,6 @@ You should have received a copy of the GNU General Public License along with this program; if not, see . - - @author: zoidberg """ import re @@ -28,6 +26,7 @@ class XFSPAccount(Account): __name__ = "XFSPAccount" __version__ = "0.06" __type__ = "account" + __description__ = """XFileSharingPro base account plugin""" __author_name__ = "zoidberg" __author_mail__ = "zoidberg@mujmail.cz" @@ -39,6 +38,7 @@ class XFSPAccount(Account): LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|>Error<' PREMIUM_PATTERN = r'>Renew premium<' + def loadAccountInfo(self, user, req): html = req.load(self.MAIN_PAGE + "?op=my_account", decode=True) -- cgit v1.2.3 From 31449f9e22cf4b5ae0f147a46bcbbf6f53a098a5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 14 Jul 2014 02:27:08 +0200 Subject: Fix class definition --- module/plugins/internal/CaptchaService.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index c2c66ed31..b2f73e4ea 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -4,7 +4,7 @@ import re from random import random -class CaptchaService(): +class CaptchaService: __name__ = "CaptchaService" __version__ = "0.05" @@ -17,7 +17,7 @@ class CaptchaService(): self.plugin = plugin -class ReCaptcha(): +class ReCaptcha: RECAPTCHA_KEY_PATTERN = r"https?://(?:www\.)?google\.com/recaptcha/api/challenge\?k=(?P\w+)" RECAPTCHA_KEY_AJAX_PATTERN = r"Recaptcha\.create\s*\(\s*[\"'](?P\w+)[\"']\s*," -- cgit v1.2.3 From 5060e4c6374a5116d0d8b02528f910f8c5f8bcf9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 15 Jul 2014 16:25:41 +0200 Subject: Fix code indentation, some bad whitespaces and missing authors + use 'not' instead 'is None' + replace __pattern__'s r" with r' + other minor cosmetics --- module/plugins/internal/CaptchaService.py | 1 + module/plugins/internal/DeadHoster.py | 2 ++ module/plugins/internal/SimpleHoster.py | 4 +++- 3 files changed, 6 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index b2f73e4ea..d8306bbb7 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -60,6 +60,7 @@ class ReCaptcha: class AdsCaptcha(CaptchaService): + def challenge(self, src): js = self.plugin.req.load(src, cookies=True) diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index 77ee7c74f..a7cf1f3ea 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -4,8 +4,10 @@ from module.plugins.Hoster import Hoster as _Hoster def create_getInfo(plugin): + def getInfo(urls): yield [('#N/A: ' + url, 0, 1, url) for url in urls] + return getInfo diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e9ba59c0d..ba143f4d8 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -53,7 +53,7 @@ def parseHtmlForm(attr_str, html, input_names=None): name = parseHtmlTagAttrValue("name", inputtag.group(1)) if name: value = parseHtmlTagAttrValue("value", inputtag.group(1)) - if value is None: + if not value: inputs[name] = inputtag.group(3) or '' else: inputs[name] = value @@ -135,6 +135,7 @@ def parseFileInfo(self, url='', html=''): def create_getInfo(plugin): + def getInfo(urls): for url in urls: cj = CookieJar(plugin.__name__) @@ -152,6 +153,7 @@ def timestamp(): class PluginParseError(Exception): + def __init__(self, msg): Exception.__init__(self) self.value = 'Parse error (%s) - plugin may be out of date' % msg -- cgit v1.2.3 From 7b8c458cca7d21a029620f98e453f746fce69cd1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 14 Jul 2014 16:10:01 +0200 Subject: Prefer single quote for dict key name --- module/plugins/internal/MultiHoster.py | 18 +++++++++--------- module/plugins/internal/SimpleHoster.py | 2 +- module/plugins/internal/UnRar.py | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 0dea76109..46d943bd7 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -141,8 +141,8 @@ class MultiHoster(Hook): self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) for hoster in self.supported: dict = self.core.pluginManager.hosterPlugins[hoster] - dict["new_module"] = module - dict["new_name"] = self.__name__ + dict['new_module'] = module + dict['new_name'] = self.__name__ if excludedList: self.logInfo("The following hosters were not overwritten - account exists: %s" % ", ".join(sorted(excludedList))) @@ -158,17 +158,17 @@ class MultiHoster(Hook): self.logDebug("Regexp: %s" % regexp) dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict["pattern"] = regexp - dict["re"] = re.compile(regexp) + dict['pattern'] = regexp + dict['re'] = re.compile(regexp) def unloadHoster(self, hoster): dict = self.core.pluginManager.hosterPlugins[hoster] if "module" in dict: - del dict["module"] + del dict['module'] if "new_module" in dict: - del dict["new_module"] - del dict["new_name"] + del dict['new_module'] + del dict['new_name'] def unload(self): """Remove override for all hosters. Scheduler job is removed by hookmanager""" @@ -178,8 +178,8 @@ class MultiHoster(Hook): # reset pattern klass = getattr(self.core.pluginManager.getPlugin(self.__name__), self.__name__) dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict["pattern"] = getattr(klass, '__pattern__', r"^unmatchable$") - dict["re"] = re.compile(dict["pattern"]) + dict['pattern'] = getattr(klass, "__pattern__", r'^unmatchable$') + dict['re'] = re.compile(dict['pattern']) def downloadFailed(self, pyfile): """remove plugin override if download fails but not if file is offline/temp.offline""" diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ba143f4d8..0dfecf0fb 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -291,7 +291,7 @@ class SimpleHoster(Hoster): return parseHtmlForm(attr_str, self.html, input_names) def checkTrafficLeft(self): - traffic = self.account.getAccountInfo(self.user, True)["trafficleft"] + traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] if traffic == -1: return True size = self.pyfile.size / 1024 diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 28f801f3e..0251620ab 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -203,8 +203,8 @@ class UnRar(AbtractExtractor): args.append("-y") # set a password - if "password" in kwargs and kwargs["password"]: - args.append("-p%s" % kwargs["password"]) + if "password" in kwargs and kwargs['password']: + args.append("-p%s" % kwargs['password']) else: args.append("-p-") -- cgit v1.2.3 From 8e47b0de30a25d0fd5dfb518bfe4e1e7beff93fd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 15 Jul 2014 16:27:44 +0200 Subject: Key attributes cleanup for account, container and crypter plugins --- module/plugins/internal/CaptchaService.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index d8306bbb7..6586256f7 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -79,6 +79,7 @@ class AdsCaptcha(CaptchaService): class SolveMedia(CaptchaService): + def challenge(self, src): html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript?k=%s" % src, cookies=True) try: -- cgit v1.2.3 From 05d258d98dd8c2faf0b769840fa1e3c4acccdce8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 20 Jul 2014 03:25:14 +0200 Subject: Fix and improve 5060e4c6374a5116d0d8b02528f910f8c5f8bcf9 --- module/plugins/internal/CaptchaService.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 6586256f7..d325d7928 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -29,7 +29,7 @@ class ReCaptcha: def detect_key(self, html): m = re.search(self.RECAPTCHA_KEY_PATTERN, html) - if not m: + if m is None: m = re.search(self.RECAPTCHA_KEY_AJAX_PATTERN, html) if m: self.recaptcha_key = m.group('key') @@ -38,9 +38,9 @@ class ReCaptcha: return None def challenge(self, key=None): - if not key and self.recaptcha_key: + if key is None and self.recaptcha_key: key = self.recaptcha_key - elif not (key or self.recaptcha_key): + else: raise TypeError("ReCaptcha key not found") js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={"k": key}, cookies=True) -- cgit v1.2.3 From 9395182da7afed55a29bde1c7cbefe4204e783f0 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 20 Jul 2014 03:02:09 +0200 Subject: Store all re.search/match object as "m" instead "found" --- module/plugins/internal/XFSPAccount.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index b58eaf15c..a1cba9af3 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -45,19 +45,19 @@ class XFSPAccount(Account): validuntil = trafficleft = None premium = True if re.search(self.PREMIUM_PATTERN, html) else False - found = re.search(self.VALID_UNTIL_PATTERN, html) - if found: + m = re.search(self.VALID_UNTIL_PATTERN, html) + if m: premium = True trafficleft = -1 try: - self.logDebug(found.group(1)) - validuntil = mktime(strptime(found.group(1), "%d %B %Y")) + self.logDebug(m.group(1)) + validuntil = mktime(strptime(m.group(1), "%d %B %Y")) except Exception, e: self.logError(e) else: - found = re.search(self.TRAFFIC_LEFT_PATTERN, html) - if found: - trafficleft = found.group(1) + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if m: + trafficleft = m.group(1) if "Unlimited" in trafficleft: premium = True else: -- cgit v1.2.3 From ba916633f2bedb04c7358000b91aed69f52e8e43 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 1 Aug 2014 19:35:59 +0200 Subject: Remove trailing whitespaces + remove license headers + import urllib methods directly + sort and fix key attributes + use save_join instead join + sort some import declarations + other minor code cosmetics --- module/plugins/internal/AbstractExtractor.py | 1 - module/plugins/internal/CaptchaService.py | 1 + module/plugins/internal/DeadCrypter.py | 2 +- module/plugins/internal/DeadHoster.py | 2 +- module/plugins/internal/MultiHoster.py | 2 +- module/plugins/internal/SimpleCrypter.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 25 ++++++------------------- module/plugins/internal/UnRar.py | 25 ++++++------------------- module/plugins/internal/UnZip.py | 17 +---------------- module/plugins/internal/XFSPAccount.py | 19 +++---------------- 10 files changed, 22 insertions(+), 76 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index c7f54f847..d1d1a09cb 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- - class ArchiveError(Exception): pass diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index d325d7928..b247ba654 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import re + from random import random diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index d19731cb5..296de739d 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -5,8 +5,8 @@ from module.plugins.Crypter import Crypter as _Crypter class DeadCrypter(_Crypter): __name__ = "DeadCrypter" - __version__ = "0.01" __type__ = "crypter" + __version__ = "0.01" __pattern__ = None diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index a7cf1f3ea..35a948824 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -13,8 +13,8 @@ def create_getInfo(plugin): class DeadHoster(_Hoster): __name__ = "DeadHoster" - __version__ = "0.11" __type__ = "hoster" + __version__ = "0.11" __pattern__ = None diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 46d943bd7..f6d202ee9 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -2,8 +2,8 @@ import re -from module.utils import remove_chars from module.plugins.Hook import Hook +from module.utils import remove_chars class MultiHoster(Hook): diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index dc93bef4d..0b99feb42 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -3,14 +3,14 @@ import re from module.plugins.Crypter import Crypter -from module.utils import html_unescape from module.plugins.internal.SimpleHoster import PluginParseError, replace_patterns, set_cookies +from module.utils import html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" - __version__ = "0.10" __type__ = "crypter" + __version__ = "0.10" __pattern__ = None diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 0dfecf0fb..23b8a5292 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -1,27 +1,14 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . -""" -from urlparse import urlparse import re + from time import time +from urlparse import urlparse -from module.plugins.Hoster import Hoster -from module.utils import html_unescape, fixup, parseFileSize -from module.network.RequestFactory import getURL from module.network.CookieJar import CookieJar +from module.network.RequestFactory import getURL +from module.plugins.Hoster import Hoster +from module.utils import fixup, html_unescape, parseFileSize def replace_patterns(string, ruleslist): @@ -164,8 +151,8 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" - __version__ = "0.34" __type__ = "hoster" + __version__ = "0.34" __pattern__ = None diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 0251620ab..19c278735 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -1,29 +1,15 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . -""" - import os import re -from os.path import join + from glob import glob -from subprocess import Popen, PIPE +from os.path import join from string import digits +from subprocess import Popen, PIPE -from module.utils import save_join, decode from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError +from module.utils import save_join, decode class UnRar(AbtractExtractor): @@ -34,6 +20,8 @@ class UnRar(AbtractExtractor): __author_name__ = "RaNaN" __author_mail__ = "RaNaN@pyload.org" + CMD = "unrar" + # there are some more uncovered rar formats re_version = re.compile(r"(UNRAR 5[\.\d]+(.*?)freeware)") re_splitfile = re.compile(r"(.*)\.part(\d+)\.rar$", re.I) @@ -41,7 +29,6 @@ class UnRar(AbtractExtractor): re_filelist = re.compile(r"(.+)\s+(\d+)\s+(\d+)\s+") re_filelist5 = re.compile(r"(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)") re_wrongpwd = re.compile("(Corrupt file or wrong password|password incorrect)", re.I) - CMD = "unrar" @staticmethod diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index bace76789..e339434f9 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -1,22 +1,7 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . -""" - -import zipfile import sys +import zipfile from module.plugins.internal.AbstractExtractor import AbtractExtractor diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index a1cba9af3..5c0bfc893 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -1,22 +1,9 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . -""" - import re + from time import mktime, strptime + from module.plugins.Account import Account from module.plugins.internal.SimpleHoster import parseHtmlForm from module.utils import parseFileSize @@ -24,8 +11,8 @@ from module.utils import parseFileSize class XFSPAccount(Account): __name__ = "XFSPAccount" - __version__ = "0.06" __type__ = "account" + __version__ = "0.06" __description__ = """XFileSharingPro base account plugin""" __author_name__ = "zoidberg" -- cgit v1.2.3 From a2b1a203a7666052d4857620b4712e82160747c2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 4 Sep 2014 22:29:59 +0200 Subject: [SimpleHoster] Improve parseFileInfo --- module/plugins/internal/SimpleHoster.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 23b8a5292..ec9cf1b70 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -83,10 +83,12 @@ def parseFileInfo(self, url='', html=''): if hasattr(self, "html"): self.html = html - if (hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html)) or \ - (hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html)): - # File offline + if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html): info['status'] = 1 + elif hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html): #@TODO: Remove in 0.4.10 + info['status'] = 1 + elif hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, html): + info['status'] = 6 else: online = False try: @@ -152,7 +154,7 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.34" + __version__ = "0.35" __pattern__ = None @@ -173,8 +175,6 @@ class SimpleHoster(Hoster): OFFLINE_PATTERN: Checks if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' - or: - FILE_OFFLINE_PATTERN: Deprecated TEMP_OFFLINE_PATTERN: Checks if the file is temporarily offline example: TEMP_OFFLINE_PATTERN = r'Server maintainance' @@ -227,13 +227,13 @@ class SimpleHoster(Hoster): def getFileInfo(self): self.logDebug("URL: %s" % self.pyfile.url) - if hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, self.html): - self.tempOffline() name, size, status = parseFileInfo(self)[:3] if status == 1: self.offline() + elif status == 6: + self.tempOffline() elif status != 2: self.logDebug(self.file_info) self.parseError('File info') -- cgit v1.2.3 From 85b5a8efcc205d9739fc1a54481cb65bf18e4d81 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 24 Sep 2014 19:16:12 +0200 Subject: [SimpleCrypter] Better built-in info --- module/plugins/internal/SimpleCrypter.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 0b99feb42..3cd52d9f1 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -24,7 +24,7 @@ class SimpleCrypter(Crypter): LINK_PATTERN: group(1) must be a download link or a regex to catch more links example: LINK_PATTERN = r'' - RECAPTCHA_PATTERN = None - SOLVEMEDIA_PATTERN = None - - ERROR_PATTERN = r'(?:class=["\']err["\'][^>]*>|<[Cc]enter>)(.+?)(?:["\']|", " ", self.errmsg)) - - if 'wait' in self.errmsg: - wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in - re.findall(r'(\d+)\s*(hour|minute|second)', self.errmsg)]) - self.wait(wait_time, True) - - elif 'captcha' in self.errmsg: - self.invalidCaptcha() - - elif 'premium' in self.errmsg and 'require' in self.errmsg: - self.fail(_("File can be downloaded by premium users only")) - - elif 'limit' in self.errmsg: - self.wait(1 * 60 * 60, True) - self.retry(25, reason="Download limit exceeded") - - elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: - self.retry(reason=_("Link expired")) - - elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: - self.tempOffline() - - elif 'download files up to' in self.errmsg: - self.fail(_("File too large for free download")) - - else: - self.fail(self.errmsg) - else: - self.errmsg = None - - return self.errmsg - - - def getPostParameters(self): - for _i in xrange(3): - self.checkErrors() - - if hasattr(self, "FORM_PATTERN"): - action, inputs = self.parseHtmlForm(self.FORM_PATTERN) - else: - action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) - - if not inputs: - action, inputs = self.parseHtmlForm('F1') - if not inputs: - if self.errmsg: - self.retry(reason=self.errmsg) - else: - self.error(_("Form not found")) - - self.logDebug(self.HOSTER_NAME, inputs) - - if 'op' in inputs and inputs['op'] in ("download2", "download3"): - if "password" in inputs: - if self.passwords: - inputs['password'] = self.passwords.pop(0) - else: - self.fail(_("No or invalid passport")) - - if not self.premium: - m = re.search(self.WAIT_PATTERN, self.html) - if m: - wait_time = int(m.group(1)) - self.setWait(wait_time, False) - else: - wait_time = 0 - - self.captcha = self.handleCaptcha(inputs) - - if wait_time: - self.wait() - - self.errmsg = None - return inputs - - else: - inputs['referer'] = self.pyfile.url - - if self.premium: - inputs['method_premium'] = "Premium Download" - if 'method_free' in inputs: - del inputs['method_free'] - else: - inputs['method_free'] = "Free Download" - if 'method_premium' in inputs: - del inputs['method_premium'] - - self.html = self.load(self.pyfile.url, post=inputs, ref=True) - self.errmsg = None - - else: - self.error(_("FORM: %s") % (inputs['op'] if 'op' in inputs else _("UNKNOWN"))) - - - def handleCaptcha(self, inputs): - m = re.search(self.CAPTCHA_PATTERN, self.html) - if m: - captcha_url = m.group(1) - inputs['code'] = self.decryptCaptcha(captcha_url) - return 1 - - m = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.S) - if m: - captcha_div = m.group(1) - self.logDebug(captcha_div) - numerals = re.findall(r'(\d)', html_unescape(captcha_div)) - inputs['code'] = "".join([a[1] for a in sorted(numerals, key=lambda num: int(num[0]))]) - self.logDebug("CAPTCHA", inputs['code'], numerals) - return 2 - - recaptcha = ReCaptcha(self) - try: - captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) - except: - captcha_key = recaptcha.detect_key() - - if captcha_key: - self.logDebug("ReCaptcha key: %s" % captcha_key) - inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) - return 3 - - solvemedia = SolveMedia(self) - try: - captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) - except: - captcha_key = solvemedia.detect_key() - - if captcha_key: - self.logDebug("SolveMedia key: %s" % captcha_key) - inputs['adcopy_challenge'], inputs['adcopy_response'] = solvemedia.challenge(captcha_key) - return 4 - - return 0 -- cgit v1.2.3 From 4ec11afb741149ea86e6ba647862b461d54c747c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Nov 2014 22:37:19 +0100 Subject: Update internal Simple plugins --- module/plugins/internal/SimpleCrypter.py | 18 +++++---- module/plugins/internal/SimpleHoster.py | 67 +++++++++++++++++++------------- 2 files changed, 50 insertions(+), 35 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 86e8077e7..1690b50c2 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -32,8 +32,8 @@ class SimpleCrypter(Crypter): LINK_PATTERN: group(1) must be a download link or a regex to catch more links example: LINK_PATTERN = r'
Files of: ([^<]+) folder' + NAME_PATTERN: (optional) group(1) should be the folder name or the webpage title + example: NAME_PATTERN = r'Files of: ([^<]+) folder' OFFLINE_PATTERN: (optional) Checks if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' @@ -59,7 +59,7 @@ class SimpleCrypter(Crypter): LINK_PATTERN = None - TITLE_REPLACEMENTS = [("&#?\w+;", fixup)] + NAME_REPLACEMENTS = [("&#?\w+;", fixup)] URL_REPLACEMENTS = [] TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct @@ -94,7 +94,8 @@ class SimpleCrypter(Crypter): self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + if not self.html: + self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) if isinstance(self.TEXT_ENCODING, basestring): self.html = unicode(self.html, self.TEXT_ENCODING) @@ -103,6 +104,9 @@ class SimpleCrypter(Crypter): def decrypt(self, pyfile): self.prepare() + if self.html is None: + self.fail(_("No html retrieved")) + self.checkOnline() package_name, folder_name = self.getPackageNameAndFolder() @@ -134,10 +138,10 @@ class SimpleCrypter(Crypter): def getPackageNameAndFolder(self): - if hasattr(self, 'TITLE_PATTERN'): + if hasattr(self, 'NAME_PATTERN'): try: - m = re.search(self.TITLE_PATTERN, self.html) - name = replace_patterns(m.group(1).strip(), self.TITLE_REPLACEMENTS) + m = re.search(self.NAME_PATTERN, self.html) + name = replace_patterns(m.group(1).strip(), self.NAME_REPLACEMENTS) folder = html_unescape(name) except: pass diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1d85dc01b..749cdf557 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -90,19 +90,20 @@ def parseHtmlForm(attr_str, html, input_names=None): def parseFileInfo(self, url="", html=""): + info = {'name': url, 'size': 0, 'status': 3} + if not url and hasattr(self, "pyfile"): url = self.pyfile.url - info = {"name": url, "size": 0, "status": 3} - if not html: if url: - return tuple(create_getInfo(self)([url])) + return next(create_getInfo(self)([url])) + + elif hasattr(self, "req") and self.req.http.code == '404': + info['status'] = 1 + elif hasattr(self, "html"): - if hasattr(self, "req") and self.req.http.code == '404': - info['status'] = 1 - else: - html = self.html + html = self.html if html: if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html): @@ -121,7 +122,8 @@ def parseFileInfo(self, url="", html=""): except: pass - for pattern in ("FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): + for pattern in ("INFO_PATTERN", "NAME_PATTERN", "SIZE_PATTERN", + "FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): #@TODO: Remove in 0.4.10 try: info.update(re.search(getattr(self, pattern), html).groupdict()) online = True @@ -133,17 +135,21 @@ def parseFileInfo(self, url="", html=""): info['status'] = 2 if 'N' in info: - info['name'] = replace_patterns(info['N'].strip(), self.FILE_NAME_REPLACEMENTS) + info['name'] = replace_patterns(info['N'].strip(), + self.FILE_NAME_REPLACEMENTS if hasattr(self, "FILE_NAME_REPLACEMENTS") else self.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 if 'S' in info: size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], - self.FILE_SIZE_REPLACEMENTS) + self.FILE_SIZE_REPLACEMENTS if hasattr(self, "FILE_SIZE_REPLACEMENTS") else self.SIZE_REPLACEMENTS) #@TODO: Remove FILE_SIZE_REPLACEMENTS check in 0.4.10 info['size'] = parseFileSize(size) elif isinstance(info['size'], basestring): unit = info['units'] if 'units' in info else None info['size'] = parseFileSize(info['size'], unit) + if not hasattr(self, "html") or self.html is None: + self.html = html + if not hasattr(self, "file_info"): self.file_info = {} @@ -162,7 +168,10 @@ def create_getInfo(plugin): else: cj = None - if hasattr(plugin, "FILE_URL_REPLACEMENTS"): + if hasattr(plugin, "URL_REPLACEMENTS"): + url = replace_patterns(url, plugin.URL_REPLACEMENTS) + + elif hasattr(plugin, "FILE_URL_REPLACEMENTS"): #@TODO: Remove in 0.4.10 url = replace_patterns(url, plugin.FILE_URL_REPLACEMENTS) if hasattr(plugin, "TEXT_ENCODING"): @@ -198,13 +207,13 @@ class SimpleHoster(Hoster): """ Following patterns should be defined by each hoster: - FILE_INFO_PATTERN: (optional) Name and Size of the file - example: FILE_INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' + INFO_PATTERN: (optional) Name and Size of the file + example: INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' or - FILE_NAME_PATTERN: (optional) Name that will be set for the file - example: FILE_NAME_PATTERN = r'(?Pfile_name)' - FILE_SIZE_PATTERN: (optional) Size that will be checked for the file - example: FILE_SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' + NAME_PATTERN: (optional) Name that will be set for the file + example: NAME_PATTERN = r'(?Pfile_name)' + SIZE_PATTERN: (optional) Size that will be checked for the file + example: SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' OFFLINE_PATTERN: (optional) Checks if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' @@ -225,9 +234,9 @@ class SimpleHoster(Hoster): example: LINK_PREMIUM_PATTERN = r'
Date: Mon, 3 Nov 2014 08:18:32 +0100 Subject: [SimpleHoster] Fix missing FOLLOWLOCATION import --- module/plugins/internal/SimpleHoster.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 749cdf557..feb373541 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -6,6 +6,8 @@ from time import time from traceback import print_exc from urlparse import urlparse +from pycurl import FOLLOWLOCATION + from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster @@ -193,7 +195,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.46" + __version__ = "0.47" __pattern__ = None -- cgit v1.2.3 From c8532b09917647b656d650e0b4a82de6022d85a4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Nov 2014 12:05:36 +0100 Subject: [XFSHoster] Fix handleOverriden --- module/plugins/internal/XFSHoster.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index b7a407a19..838af9102 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -2,9 +2,10 @@ import re -from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME from random import random +from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME + from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia from module.plugins.internal.SimpleHoster import create_getInfo, replace_patterns, set_cookies, SimpleHoster from module.plugins.Plugin import Fail @@ -14,7 +15,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.08" + __version__ = "0.09" __pattern__ = None @@ -71,6 +72,12 @@ class XFSHoster(SimpleHoster): self.errmsg = None self.passwords = self.getPassword().splitlines() + if re.match(self.__pattern__, self.pyfile.url) is None: + if self.premium: + self.handleOverriden() + else: + self.fail(_("Only premium users can download from other hosters")) + return super(XFSHoster, self).prepare() @@ -118,26 +125,31 @@ class XFSHoster(SimpleHoster): return m.group(1) - #@TODO: Re-enable def handleOverriden(self): #only tested with easybytez.com - self.html = self.load("http://www.%s/" % self.HOSTER_NAME) + self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) + action, inputs = self.parseHtmlForm('') + upload_id = "%012d" % int(random() * 10 ** 12) action += upload_id + "&js_on=1&utype=prem&upload_type=url" + inputs['tos'] = '1' inputs['url_mass'] = self.pyfile.url inputs['up1oad_type'] = 'url' self.logDebug(action, inputs) - #wait for file to upload to easybytez.com - self.req.http.c.setopt(LOW_SPEED_TIME, 600) + + self.req.http.c.setopt(LOW_SPEED_TIME, 600) #: wait for file to upload to easybytez.com + self.html = self.load(action, post=inputs) action, inputs = self.parseHtmlForm('F1') if not inputs: - self.error(_("TEXTAREA not found")) + self.error(_("TEXTAREA F1 not found")) + self.logDebug(inputs) + if inputs['st'] == 'OK': self.html = self.load(action, post=inputs) elif inputs['st'] == 'Can not leech file': @@ -212,7 +224,7 @@ class XFSHoster(SimpleHoster): if self.errmsg: self.retry(reason=self.errmsg) else: - self.error(_("Form not found")) + self.error(_("TEXTAREA F1 not found")) self.logDebug(inputs) -- cgit v1.2.3 From 8e921a978e79d93da3036ce3bdc1fb176c74fabb Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Nov 2014 12:09:04 +0100 Subject: Spare code cosmetics --- module/plugins/internal/MultiHoster.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 3596641b0..24942f8a3 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -16,13 +16,14 @@ class MultiHoster(Hook): __authors__ = [("pyLoad Team", "admin@pyload.org")] - replacements = [("2shared.com", "twoshared.com"), ("4shared.com", "fourshared.com"), ("cloudnator.com", "shragle.com"), - ("ifile.it", "filecloud.io"), ("easy-share.com", "crocko.com"), ("freakshare.net", "freakshare.com"), - ("hellshare.com", "hellshare.cz"), ("share-rapid.cz", "sharerapid.com"), ("sharerapid.cz", "sharerapid.com"), - ("ul.to", "uploaded.to"), ("uploaded.net", "uploaded.to"), ("1fichier.com", "onefichier.com")] - ignored = [] interval = 24 * 60 * 60 #: reload hosters daily + HOSTER_REPLACEMENTS = [("2shared.com", "twoshared.com"), ("4shared.com", "fourshared.com"), ("cloudnator.com", "shragle.com"), + ("ifile.it", "filecloud.io"), ("easy-share.com", "crocko.com"), ("freakshare.net", "freakshare.com"), + ("hellshare.com", "hellshare.cz"), ("share-rapid.cz", "sharerapid.com"), ("sharerapid.cz", "sharerapid.com"), + ("ul.to", "uploaded.to"), ("uploaded.net", "uploaded.to"), ("1fichier.com", "onefichier.com")] + HOSTER_EXCLUDED = [] + def setup(self): self.hosters = [] @@ -42,7 +43,7 @@ class MultiHoster(Hook): if not self.hosters: try: - hosterSet = self.toHosterSet(self.getHoster()) - set(self.ignored) + hosterSet = self.toHosterSet(self.getHoster()) - set(self.HOSTER_EXCLUDED) except Exception, e: self.logError(e) return [] @@ -68,7 +69,7 @@ class MultiHoster(Hook): def toHosterSet(self, hosters): hosters = set((str(x).strip().lower() for x in hosters)) - for rep in self.replacements: + for rep in self.HOSTER_REPLACEMENTS: if rep[0] in hosters: hosters.remove(rep[0]) hosters.add(rep[1]) -- cgit v1.2.3 From 4666843d7e7b90e409019b471bcf35c67c4df645 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Nov 2014 12:12:09 +0100 Subject: [SimpleHoster] Fix parseFileInfo and improve getFileInfo --- module/plugins/internal/SimpleHoster.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index feb373541..b9ac84783 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -92,14 +92,14 @@ def parseHtmlForm(attr_str, html, input_names=None): def parseFileInfo(self, url="", html=""): - info = {'name': url, 'size': 0, 'status': 3} - if not url and hasattr(self, "pyfile"): url = self.pyfile.url + info = {'name': url, 'size': 0, 'status': 3} + if not html: if url: - return next(create_getInfo(self)([url])) + return create_getInfo(self)([url]).next() elif hasattr(self, "req") and self.req.http.code == '404': info['status'] = 1 @@ -155,8 +155,12 @@ def parseFileInfo(self, url="", html=""): if not hasattr(self, "file_info"): self.file_info = {} + self.logDebug(_("File info (before update): %s") % self.file_info) + self.file_info.update(info) + self.logDebug(_("File info (after update): %s") % self.file_info) + return info['name'], info['size'], info['status'], url @@ -195,7 +199,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.47" + __version__ = "0.48" __pattern__ = None @@ -245,10 +249,6 @@ class SimpleHoster(Hoster): FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - def init(self): - self.file_info = {} - - def setup(self): self.resumeDownload = self.multiDL = self.premium @@ -313,29 +313,26 @@ class SimpleHoster(Hoster): def getFileInfo(self): - name, size, status = parseFileInfo(self)[:3] + name, size, status, url = parseFileInfo(self) - msg = _("File info: %s") % self.file_info - self.logDebug(msg) + if name and name != url: + self.pyfile.name = name + else: + self.pyfile.name = self.file_info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) if status == 1: self.offline() elif status == 6: self.tempOffline() elif status != 2: - self.error(msg) - - if name: - self.pyfile.name = name - else: - self.pyfile.name = html_unescape(urlparse(self.pyfile.url).path.split("/")[-1]) + self.error(_("File info: %s") % self.file_info) if size: self.pyfile.size = size else: self.logError(_("File size not parsed")) - self.logDebug("FILE NAME: %s" % self.pyfile.name, "FILE SIZE: %d" % self.pyfile.size) + self.logDebug("FILE NAME: %s" % self.pyfile.name, "FILE SIZE: %d" % self.pyfile.size or _("Unknown")) return self.file_info -- cgit v1.2.3 From cb598e72d1b8bee0302c21e09781b6b2389593c1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Nov 2014 12:12:55 +0100 Subject: [SimpleCrypter] Use getFileInfo --- module/plugins/internal/SimpleCrypter.py | 67 ++++++++++++++------------------ 1 file changed, 29 insertions(+), 38 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 1690b50c2..9856ae2e5 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -6,14 +6,14 @@ from traceback import print_exc from module.plugins.Crypter import Crypter from module.plugins.Plugin import Fail -from module.plugins.internal.SimpleHoster import _error, _wait, replace_patterns, set_cookies +from module.plugins.internal.SimpleHoster import _error, _wait, parseFileInfo, replace_patterns, set_cookies from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.24" + __version__ = "0.25" __pattern__ = None __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -32,8 +32,8 @@ class SimpleCrypter(Crypter): LINK_PATTERN: group(1) must be a download link or a regex to catch more links example: LINK_PATTERN = r'
Files of: ([^<]+) folder' + NAME_PATTERN: (optional) folder name or webpage title + example: NAME_PATTERN = r'Files of: (?P<N>[^<]+) folder' OFFLINE_PATTERN: (optional) Checks if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' @@ -84,10 +84,10 @@ class SimpleCrypter(Crypter): def prepare(self): if self.LOGIN_ACCOUNT and not self.account: - self.fail(_("Required account not found!")) + self.fail(_("Required account not found")) if self.LOGIN_PREMIUM and not self.premium: - self.fail(_("Required premium account not found!")) + self.fail(_("Required premium account not found")) if isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) @@ -107,53 +107,44 @@ class SimpleCrypter(Crypter): if self.html is None: self.fail(_("No html retrieved")) - self.checkOnline() + info = self.getFileInfo() - package_name, folder_name = self.getPackageNameAndFolder() - - self.package_links = self.getLinks() + self.links = self.getLinks() if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): self.handleMultiPages() - self.logDebug("Package has %d links" % len(self.package_links)) + self.logDebug("Package has %d links" % len(self.links)) - if self.package_links: - self.packages = [(package_name, self.package_links, folder_name)] + if self.links: + self.packages = [(info['name'], self.links, info['folder'])] - def getLinks(self): - """ - Returns the links extracted from self.html - You should override this only if it's impossible to extract links using only the LINK_PATTERN. - """ - return re.findall(self.LINK_PATTERN, self.html) + def getFileInfo(self): + name, size, status, url = parseFileInfo(self) + if name and name != url: + self.pyfile.name = name + else: + self.pyfile.name = self.file_info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) - def checkOnline(self): - if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, self.html): + if status == 1: self.offline() - elif hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, self.html): + elif status == 6: self.tempOffline() + self.file_info['folder'] = self.pyfile.name - def getPackageNameAndFolder(self): - if hasattr(self, 'NAME_PATTERN'): - try: - m = re.search(self.NAME_PATTERN, self.html) - name = replace_patterns(m.group(1).strip(), self.NAME_REPLACEMENTS) - folder = html_unescape(name) - except: - pass - else: - self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder)) - return name, folder + self.logDebug("FILE NAME: %s" % self.pyfile.name) + return self.file_info - name = self.pyfile.package().name - folder = self.pyfile.package().folder - self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder)) - return name, folder + def getLinks(self): + """ + Returns the links extracted from self.html + You should override this only if it's impossible to extract links using only the LINK_PATTERN. + """ + return re.findall(self.LINK_PATTERN, self.html) def handleMultiPages(self): @@ -165,7 +156,7 @@ class SimpleCrypter(Crypter): for p in xrange(2, pages + 1): self.html = self.loadPage(p) - self.package_links += self.getLinks() + self.links += self.getLinks() #@TODO: Remove in 0.4.10 -- cgit v1.2.3 From d7729f9020bc5959f295ca5efb75f3ce0dd605bb Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Nov 2014 15:46:50 +0100 Subject: [SimpleHoster] Fix file_info stuff --- module/plugins/internal/SimpleHoster.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index b9ac84783..1555151a5 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -152,14 +152,20 @@ def parseFileInfo(self, url="", html=""): if not hasattr(self, "html") or self.html is None: self.html = html - if not hasattr(self, "file_info"): - self.file_info = {} + if not hasattr(self, "info"): + self.info = {} - self.logDebug(_("File info (before update): %s") % self.file_info) + try: #: Remove try-except statement in 0.4.10 + self.logDebug(_("File info (before update): %s") % self.info) + except: + pass - self.file_info.update(info) + self.info.update(info) - self.logDebug(_("File info (after update): %s") % self.file_info) + try: #: Remove try-except statement in 0.4.10 + self.logDebug(_("File info (after update): %s") % self.info) + except: + pass return info['name'], info['size'], info['status'], url @@ -199,9 +205,9 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.48" + __version__ = "0.49" - __pattern__ = None + __pattern__ = r'^unmatchable$' __description__ = """Simple hoster plugin""" __license__ = "GPLv3" @@ -249,6 +255,10 @@ class SimpleHoster(Hoster): FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + def init(): + self.info = {} + + def setup(self): self.resumeDownload = self.multiDL = self.premium @@ -285,7 +295,7 @@ class SimpleHoster(Hoster): else: premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only and 'status' not in self.file_info: #: Usually premium only pages doesn't show any file information + if not premium_only and 'status' not in self.info: #: Usually premium only pages doesn't show any file information self.getFileInfo() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): @@ -318,14 +328,14 @@ class SimpleHoster(Hoster): if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.file_info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) + self.pyfile.name = self.info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) if status == 1: self.offline() elif status == 6: self.tempOffline() elif status != 2: - self.error(_("File info: %s") % self.file_info) + self.error(_("File info: %s") % self.info) if size: self.pyfile.size = size @@ -333,7 +343,7 @@ class SimpleHoster(Hoster): self.logError(_("File size not parsed")) self.logDebug("FILE NAME: %s" % self.pyfile.name, "FILE SIZE: %d" % self.pyfile.size or _("Unknown")) - return self.file_info + return self.info def handleFree(self): -- cgit v1.2.3 From 03f3b86f500c495932fd118b54569d92f700847c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Nov 2014 16:57:55 +0100 Subject: Code cosmetics about file_info and other stuff --- module/plugins/internal/DeadCrypter.py | 2 +- module/plugins/internal/DeadHoster.py | 2 +- module/plugins/internal/SimpleCrypter.py | 16 +++++++++------- module/plugins/internal/SimpleHoster.py | 12 ++++++------ module/plugins/internal/XFSCrypter.py | 2 +- module/plugins/internal/XFSHoster.py | 2 +- 6 files changed, 19 insertions(+), 17 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 91cdf6036..4c635e366 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -8,7 +8,7 @@ class DeadCrypter(_Crypter): __type__ = "crypter" __version__ = "0.02" - __pattern__ = None + __pattern__ = r'^unmatchable$' __description__ = """ Crypter is no longer available """ __license__ = "GPLv3" diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index 2d91b1d2d..b78068069 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -16,7 +16,7 @@ class DeadHoster(_Hoster): __type__ = "hoster" __version__ = "0.12" - __pattern__ = None + __pattern__ = r'^unmatchable$' __description__ = """ Hoster is no longer available """ __license__ = "GPLv3" diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 9856ae2e5..0985bac60 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -13,9 +13,9 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.25" + __version__ = "0.26" - __pattern__ = None + __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] @@ -71,6 +71,8 @@ class SimpleCrypter(Crypter): #@TODO: remove in 0.4.10 def init(self): + self.info = {} + account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") account = self.core.accountManager.getAccountPlugin(account_name) @@ -126,17 +128,17 @@ class SimpleCrypter(Crypter): if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.file_info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) + self.pyfile.name = self.info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) - if status == 1: + if status is 1: self.offline() - elif status == 6: + elif status is 6: self.tempOffline() - self.file_info['folder'] = self.pyfile.name + self.info['folder'] = self.pyfile.name self.logDebug("FILE NAME: %s" % self.pyfile.name) - return self.file_info + return self.info def getLinks(self): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1555151a5..1f673a5c2 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -155,14 +155,14 @@ def parseFileInfo(self, url="", html=""): if not hasattr(self, "info"): self.info = {} - try: #: Remove try-except statement in 0.4.10 + try: self.logDebug(_("File info (before update): %s") % self.info) except: pass self.info.update(info) - try: #: Remove try-except statement in 0.4.10 + try: self.logDebug(_("File info (after update): %s") % self.info) except: pass @@ -255,7 +255,7 @@ class SimpleHoster(Hoster): FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - def init(): + def init(self): self.info = {} @@ -330,11 +330,11 @@ class SimpleHoster(Hoster): else: self.pyfile.name = self.info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) - if status == 1: + if status is 1: self.offline() - elif status == 6: + elif status is 6: self.tempOffline() - elif status != 2: + elif status is not 2: self.error(_("File info: %s") % self.info) if size: diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 5a126707b..2e491050b 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -8,7 +8,7 @@ class XFSCrypter(SimpleCrypter): __type__ = "crypter" __version__ = "0.03" - __pattern__ = None + __pattern__ = r'^unmatchable$' __description__ = """XFileSharing decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 838af9102..a2e4f5846 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -17,7 +17,7 @@ class XFSHoster(SimpleHoster): __type__ = "hoster" __version__ = "0.09" - __pattern__ = None + __pattern__ = r'^unmatchable$' __description__ = """XFileSharing hoster plugin""" __license__ = "GPLv3" -- cgit v1.2.3 From 2f4d7d614171054c3c2b0b6185fa4ea465f5312f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 01:09:43 +0100 Subject: Use more print_exc + code cosmetics --- module/plugins/internal/SimpleHoster.py | 4 +--- module/plugins/internal/XFSHoster.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1f673a5c2..ac90e193a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -24,8 +24,6 @@ def _error(self, reason, type): msg += ": " + reason.strip() if reason else "" msg += _(" | Plugin may be out of date") - if self.core.debug: - print_exc() raise Fail(msg) @@ -315,7 +313,7 @@ class SimpleHoster(Hoster): self.req.http.c.setopt(FOLLOWLOCATION, 1) - if parseFileInfo(self, url, html)[2] != 2: + if parseFileInfo(self, url, html)[2] is not 2: try: return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1) except: diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index a2e4f5846..147c4dfde 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -102,7 +102,7 @@ class XFSHoster(SimpleHoster): def getDownloadLink(self): - for i in xrange(5): + for i in xrange(1, 5): self.logDebug("Getting download link: #%d" % i) data = self.getPostParameters() -- cgit v1.2.3 From e9376045f48255d3cbfdbaf5b5c6fbfd9112f17d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 03:23:06 +0100 Subject: [SimpleCrypter][SimpleHoster] Fix info data retrieving --- module/plugins/internal/SimpleCrypter.py | 14 ++++++----- module/plugins/internal/SimpleHoster.py | 42 +++++++++++++++++--------------- 2 files changed, 31 insertions(+), 25 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 0985bac60..a694cc2c9 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -13,7 +13,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.26" + __version__ = "0.27" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -60,10 +60,10 @@ class SimpleCrypter(Crypter): LINK_PATTERN = None NAME_REPLACEMENTS = [("&#?\w+;", fixup)] - URL_REPLACEMENTS = [] + URL_REPLACEMENTS = [] TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] + COOKIES = True #: or False or list of tuples [(domain, name, value)] LOGIN_ACCOUNT = False LOGIN_PREMIUM = False @@ -96,7 +96,7 @@ class SimpleCrypter(Crypter): self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - if not self.html: + if self.html is None: self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) if isinstance(self.TEXT_ENCODING, basestring): @@ -109,7 +109,8 @@ class SimpleCrypter(Crypter): if self.html is None: self.fail(_("No html retrieved")) - info = self.getFileInfo() + if not self.info: + self.getFileInfo() self.links = self.getLinks() @@ -119,7 +120,7 @@ class SimpleCrypter(Crypter): self.logDebug("Package has %d links" % len(self.links)) if self.links: - self.packages = [(info['name'], self.links, info['folder'])] + self.packages = [(self.info['name'], self.links, self.info['folder'])] def getFileInfo(self): @@ -132,6 +133,7 @@ class SimpleCrypter(Crypter): if status is 1: self.offline() + elif status is 6: self.tempOffline() diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ac90e193a..3de49bef9 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -147,23 +147,21 @@ def parseFileInfo(self, url="", html=""): unit = info['units'] if 'units' in info else None info['size'] = parseFileSize(info['size'], unit) - if not hasattr(self, "html") or self.html is None: + if hasattr(self, "html") and self.html is None: self.html = html - if not hasattr(self, "info"): - self.info = {} - - try: - self.logDebug(_("File info (before update): %s") % self.info) - except: - pass + if hasattr(self, "info"): + try: + self.logDebug(_("File info (before update): %s") % self.info) + except: + pass - self.info.update(info) + self.info.update(info) - try: - self.logDebug(_("File info (after update): %s") % self.info) - except: - pass + try: + self.logDebug(_("File info (after update): %s") % self.info) + except: + pass return info['name'], info['size'], info['status'], url @@ -203,7 +201,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.49" + __version__ = "0.50" __pattern__ = r'^unmatchable$' @@ -246,10 +244,10 @@ class SimpleHoster(Hoster): NAME_REPLACEMENTS = [("&#?\w+;", fixup)] SIZE_REPLACEMENTS = [] - URL_REPLACEMENTS = [] + URL_REPLACEMENTS = [] - TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] + TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct + COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account @@ -274,8 +272,11 @@ class SimpleHoster(Hoster): direct_link = self.getDirectLink(self.pyfile.url) if direct_link: return direct_link + else: + self.html = None + self.info = {} - if not self.html: + if self.html is None: self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) if isinstance(self.TEXT_ENCODING, basestring): @@ -286,6 +287,7 @@ class SimpleHoster(Hoster): direct_link = self.prepare() if isinstance(direct_link, basestring): + self.logInfo(_("Direct download link detected")) self.download(direct_link, ref=True, cookies=True, disposition=True) elif self.html is None: @@ -293,7 +295,7 @@ class SimpleHoster(Hoster): else: premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only and 'status' not in self.info: #: Usually premium only pages doesn't show any file information + if not premium_only and not self.info: #: Usually premium only pages doesn't show any file information self.getFileInfo() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): @@ -330,8 +332,10 @@ class SimpleHoster(Hoster): if status is 1: self.offline() + elif status is 6: self.tempOffline() + elif status is not 2: self.error(_("File info: %s") % self.info) -- cgit v1.2.3 From edfaaed90d59991ada592da31a65cc518d7d3871 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 03:23:39 +0100 Subject: Code cosmetics --- module/plugins/internal/MultiHoster.py | 2 +- module/plugins/internal/XFSCrypter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 24942f8a3..f0d3993a7 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -22,7 +22,7 @@ class MultiHoster(Hook): ("ifile.it", "filecloud.io"), ("easy-share.com", "crocko.com"), ("freakshare.net", "freakshare.com"), ("hellshare.com", "hellshare.cz"), ("share-rapid.cz", "sharerapid.com"), ("sharerapid.cz", "sharerapid.com"), ("ul.to", "uploaded.to"), ("uploaded.net", "uploaded.to"), ("1fichier.com", "onefichier.com")] - HOSTER_EXCLUDED = [] + HOSTER_EXCLUDED = [] def setup(self): diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 2e491050b..fb6f7fed3 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -25,5 +25,5 @@ class XFSCrypter(SimpleCrypter): LINK_PATTERN = r'<(?:td|TD) [^>]*>\s*]*>.+?(?:)?\s*' NAME_PATTERN = r'<[tT]itle>.*?\: (.+) folder' - OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' -- cgit v1.2.3 From d429c97c55e06a8f2ee7de64e4add783e80e9956 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 03:25:05 +0100 Subject: [XFSAccount] Better traffic fallback --- module/plugins/internal/XFSAccount.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 597964d4c..aa8a8ace1 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.23" + __version__ = "0.24" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -32,12 +32,12 @@ class XFSAccount(Account): HOSTER_DOMAIN = None - COOKIES = [(HOSTER_DOMAIN, "lang", "english")] #: or list of tuples [(domain, name, value)] + COOKIES = [(HOSTER_DOMAIN, "lang", "english")] VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(\d{1,2} [\w^_]+ \d{4})' TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' - TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found + TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' @@ -86,7 +86,9 @@ class XFSAccount(Account): if m: try: traffic = m.groupdict() - if "nlimited" in traffic['S']: + size = traffic['S'] + + if "nlimited" in size: trafficleft = -1 if premium is None: premium = True @@ -98,10 +100,10 @@ class XFSAccount(Account): else: unit = "" - trafficleft = self.parseTraffic(traffic['S'] + unit) + trafficleft = self.parseTraffic(size + unit) except Exception, e: - self.logDebug(str(e)) + self.logError(str(e)) return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium or False} -- cgit v1.2.3 From a476b801080244611001acca694e9937be1c9405 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 03:26:21 +0100 Subject: [XFSHoster] Improved MultiHoster check and download limit detection --- module/plugins/internal/XFSHoster.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 147c4dfde..008cfb3e1 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -6,6 +6,7 @@ from random import random from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME +from module.plugins.hoster.UnrestrictLi import secondsToMidnight from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia from module.plugins.internal.SimpleHoster import create_getInfo, replace_patterns, set_cookies, SimpleHoster from module.plugins.Plugin import Fail @@ -15,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.09" + __version__ = "0.10" __pattern__ = r'^unmatchable$' @@ -35,18 +36,18 @@ class XFSHoster(SimpleHoster): NAME_PATTERN = r'[\d\.\,]+) ?(?P[\w^_]+)?\)' - OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' WAIT_PATTERN = r'.*?>(\d+)' OVR_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' - LINK_PATTERN = None #: final download url pattern + LINK_PATTERN = None #: final download url pattern - CAPTCHA_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' + CAPTCHA_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' CAPTCHA_DIV_PATTERN = r'>Enter code.*?(.+?)
' - RECAPTCHA_PATTERN = None - SOLVEMEDIA_PATTERN = None + RECAPTCHA_PATTERN = None + SOLVEMEDIA_PATTERN = None ERROR_PATTERN = r'(?:class=["\']err["\'][^>]*>|<[Cc]enter>)(.+?)(?:["\']| Date: Tue, 4 Nov 2014 13:03:24 +0100 Subject: [OneFichierCom] Fixed account plugin --- module/plugins/internal/XFSAccount.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index aa8a8ace1..6a3b3605e 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -56,12 +56,12 @@ class XFSAccount(Account): def loadAccountInfo(self, user, req): - html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) - validuntil = None trafficleft = None premium = None + html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) + if hasattr(self, "PREMIUM_PATTERN"): premium = True if re.search(self.PREMIUM_PATTERN, html) else False -- cgit v1.2.3 From 6412e51af626ee89ce6d9dac99a6c53e1878d142 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 14:19:32 +0100 Subject: logDebug when detects ddl or multi hoster link --- module/plugins/internal/SimpleHoster.py | 2 ++ module/plugins/internal/XFSHoster.py | 1 + 2 files changed, 3 insertions(+) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3de49bef9..2b729dae4 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -269,10 +269,12 @@ class SimpleHoster(Hoster): self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 if self.premium: + self.logDebug(_("Checking for direct download link...")) direct_link = self.getDirectLink(self.pyfile.url) if direct_link: return direct_link else: + self.logDebug(_("No direct download link found")) self.html = None self.info = {} diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 008cfb3e1..12ac4845d 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -75,6 +75,7 @@ class XFSHoster(SimpleHoster): # MultiHoster check if self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern']: + self.logInfo(_("Multi hoster detected")) if self.premium: self.handleOverriden() else: -- cgit v1.2.3 From 0d1fe176a7963a71f9eb6735afc4132c26550b75 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 19:59:21 +0100 Subject: Fix missing urlparse import --- module/plugins/internal/SimpleCrypter.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index a694cc2c9..69cbb7051 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -2,7 +2,7 @@ import re -from traceback import print_exc +from urlparse import urlparse from module.plugins.Crypter import Crypter from module.plugins.Plugin import Fail @@ -13,7 +13,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.27" + __version__ = "0.28" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 2b729dae4..bd6c69db6 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -3,7 +3,6 @@ import re from time import time -from traceback import print_exc from urlparse import urlparse from pycurl import FOLLOWLOCATION -- cgit v1.2.3 From f74309f89923317eeaaca31d74dff10590ed3d40 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 23:42:46 +0100 Subject: [XFSHoster] Use URL_REPLACEMENTS to clean url --- module/plugins/internal/XFSAccount.py | 10 ++++------ module/plugins/internal/XFSHoster.py | 6 ++++-- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 6a3b3605e..6e34409ed 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -2,8 +2,8 @@ import re -from urlparse import urljoin from time import gmtime, mktime, strptime +from urlparse import urljoin from module.plugins.Account import Account from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies @@ -23,14 +23,12 @@ class XFSAccount(Account): """ Following patterns should be defined by each hoster: - HOSTER_URL: (optional) - example: HOSTER_URL = r'linestorage.com' - PREMIUM_PATTERN: (optional) Checks if the account is premium example: PREMIUM_PATTERN = r'>Renew premium' """ HOSTER_DOMAIN = None + HOSTER_URL = None COOKIES = [(HOSTER_DOMAIN, "lang", "english")] @@ -51,8 +49,8 @@ class XFSAccount(Account): # if not self.HOSTER_DOMAIN: # self.fail(_("Missing HOSTER_DOMAIN")) - if not hasattr(self, "HOSTER_URL"): - self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN.replace("www.", "", 1) + if not self.HOSTER_URL: + self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN def loadAccountInfo(self, user, req): diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 12ac4845d..4e9a0ffe6 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.10" + __version__ = "0.11" __pattern__ = r'^unmatchable$' @@ -28,7 +28,9 @@ class XFSHoster(SimpleHoster): HOSTER_DOMAIN = None - HOSTER_NAME = None + HOSTER_NAME = None + + URL_REPLACEMENTS = [(r'/(?:embed-)?(\w{12}).*', r'/\1')] #: plus support embedded files COOKIES = [(HOSTER_DOMAIN, "lang", "english")] -- cgit v1.2.3 From 8fdf56ba750f7bc513e87632cfa2f4415c9c67a5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 6 Nov 2014 03:30:39 +0100 Subject: [XFSAccount] Fix premium & validuntil detection --- module/plugins/internal/XFSAccount.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 6e34409ed..8a2242018 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.24" + __version__ = "0.25" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -20,18 +20,13 @@ class XFSAccount(Account): ("Walter Purcaro", "vuolter@gmail.com")] - """ - Following patterns should be defined by each hoster: - - PREMIUM_PATTERN: (optional) Checks if the account is premium - example: PREMIUM_PATTERN = r'>Renew premium' - """ - HOSTER_DOMAIN = None HOSTER_URL = None COOKIES = [(HOSTER_DOMAIN, "lang", "english")] + PREMIUM_PATTERN = r'\(Premium only\)' + VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(\d{1,2} [\w^_]+ \d{4})' TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' @@ -60,8 +55,7 @@ class XFSAccount(Account): html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) - if hasattr(self, "PREMIUM_PATTERN"): - premium = True if re.search(self.PREMIUM_PATTERN, html) else False + premium = True if re.search(self.PREMIUM_PATTERN, html) else False m = re.search(self.VALID_UNTIL_PATTERN, html) if m: @@ -76,9 +70,8 @@ class XFSAccount(Account): if validuntil > mktime(gmtime()): premium = True else: - if premium is False: #: registered account type (not premium) - validuntil = -1 premium = False + validuntil = None #: registered account type (not premium) m = re.search(self.TRAFFIC_LEFT_PATTERN, html) if m: @@ -88,8 +81,8 @@ class XFSAccount(Account): if "nlimited" in size: trafficleft = -1 - if premium is None: - premium = True + if validuntil is None: + validuntil = -1 else: if 'U' in traffic: unit = traffic['U'] @@ -103,7 +96,7 @@ class XFSAccount(Account): except Exception, e: self.logError(str(e)) - return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium or False} + return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} def login(self, user, data, req): -- cgit v1.2.3 From 0997d9f86b10cf8358c58d26150bbfa95e9c80b9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 6 Nov 2014 04:35:01 +0100 Subject: [XFSCrypter] Fix a couple of patterns --- module/plugins/internal/XFSCrypter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index fb6f7fed3..1d5342137 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -6,7 +6,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.03" + __version__ = "0.04" __pattern__ = r'^unmatchable$' @@ -18,12 +18,12 @@ class XFSCrypter(SimpleCrypter): HOSTER_DOMAIN = None HOSTER_NAME = None - URL_REPLACEMENTS = [(r'[?/&]+$', r''), (r'(.+/[^?]*)$', r'\1?'), (r'$', r'&per_page=10000')] + URL_REPLACEMENTS = [(r'[?/&]+$', ""), (r'&?per_page=\d+', ""), (r'(.+/[^?]+)', r'\1\?'), (r'$', r'&per_page=10000')] COOKIES = [(HOSTER_DOMAIN, "lang", "english")] - LINK_PATTERN = r'<(?:td|TD) [^>]*>\s*]*>.+?(?:)?\s*' - NAME_PATTERN = r'<[tT]itle>.*?\: (.+) folder' + LINK_PATTERN = r'<(?:td|TD).*?>\s*.+?(?:)?\s*' + NAME_PATTERN = r'<[tT]itle>.*?\: (?P.+) folder' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' -- cgit v1.2.3 From b95777d6a6a5d4189bf20ac6f5d918501e89f0c6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 6 Nov 2014 04:43:03 +0100 Subject: [XFSCrypter] Improve URL_REPLACEMENTS --- module/plugins/internal/XFSCrypter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 1d5342137..62fd8c017 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -18,7 +18,7 @@ class XFSCrypter(SimpleCrypter): HOSTER_DOMAIN = None HOSTER_NAME = None - URL_REPLACEMENTS = [(r'[?/&]+$', ""), (r'&?per_page=\d+', ""), (r'(.+/[^?]+)', r'\1\?'), (r'$', r'&per_page=10000')] + URL_REPLACEMENTS = [(r'&?per_page=\d+', ""), (r'[?/&]+$', ""), (r'(.+/[^?]+)$', r'\1?'), (r'$', r'&per_page=10000')] COOKIES = [(HOSTER_DOMAIN, "lang", "english")] -- cgit v1.2.3 From 6cb8c84d19066db3ad32f84dff864f058b6a2be9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 6 Nov 2014 15:13:01 +0100 Subject: [SimpleHoster] Fix getDirectLink --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index bd6c69db6..2c95c8b38 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -200,7 +200,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.50" + __version__ = "0.51" __pattern__ = r'^unmatchable$' @@ -318,7 +318,7 @@ class SimpleHoster(Hoster): if parseFileInfo(self, url, html)[2] is not 2: try: - return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1) + return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1).strip() #@TODO: Remove .strip() in 0.4.10 except: pass -- cgit v1.2.3 From 7afad85d15451a235e11b9d7752d828fadbf5d85 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 6 Nov 2014 22:41:16 +0100 Subject: [XFSHoster] Fix Multi hoster detection --- module/plugins/internal/XFSHoster.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 4e9a0ffe6..b380c5f69 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.11" + __version__ = "0.12" __pattern__ = r'^unmatchable$' @@ -76,12 +76,14 @@ class XFSHoster(SimpleHoster): self.passwords = self.getPassword().splitlines() # MultiHoster check - if self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern']: + if self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] + and re.match(self.__pattern__, self.pyfile.url) is None: self.logInfo(_("Multi hoster detected")) if self.premium: + self.logDebug(_("Looking for leech download link...")) self.handleOverriden() else: - self.fail(_("Only premium users can download from other hosters")) + self.fail(_("Only premium users can use url leech feature")) return super(XFSHoster, self).prepare() @@ -155,12 +157,19 @@ class XFSHoster(SimpleHoster): self.logDebug(inputs) - if inputs['st'] == 'OK': + stmsg = inputs['st'] + + if stmsg == "OK": self.html = self.load(action, post=inputs) - elif inputs['st'] == 'Can not leech file': - self.retry(20, 3 * 60, inputs['st']) + + elif "Can not leech file" in stmsg: + self.retry(20, 3 * 60, _("Can not leech file")) + + elif "all Leech traffic today" in stmsg: + self.retry(wait_time=secondsToMidnight(gmt=2), reason=_("You've used all Leech traffic today")) + else: - self.fail(_(inputs['st'])) + self.fail(stmsg) #get easybytez.com link for uploaded file m = re.search(self.OVR_LINK_PATTERN, self.html) @@ -170,10 +179,10 @@ class XFSHoster(SimpleHoster): self.pyfile.url = m.group(1) header = self.load(self.pyfile.url, just_header=True, decode=True) - if 'location' in header: # Direct link + if 'location' in header: #: Direct download link self.download(header['location'], ref=True, cookies=True, disposition=True) else: - self.retry(reason=_("OVR link location not found")) + self.error(_("No leech download link found")) def checkErrors(self): @@ -196,13 +205,13 @@ class XFSHoster(SimpleHoster): elif 'limit' in self.errmsg: if 'days' in self.errmsg: delay = secondsToMidnight(gmt=2) - retries = 2 + retries = 3 else: delay = 1 * 60 * 60 retries = 25 self.wait(delay, True) - self.retry(retries, reason="Download limit exceeded") + self.retry(retries, reason=_("Download limit exceeded")) elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: self.retry(reason=_("Link expired")) -- cgit v1.2.3 From 6bcfdef32e7e62f8f749ceb6618ee7550bce28de Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 6 Nov 2014 22:42:59 +0100 Subject: Lowercase url to load --- module/plugins/internal/SimpleHoster.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 2c95c8b38..87a348367 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -200,7 +200,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.51" + __version__ = "0.52" __pattern__ = r'^unmatchable$' @@ -268,7 +268,7 @@ class SimpleHoster(Hoster): self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 if self.premium: - self.logDebug(_("Checking for direct download link...")) + self.logDebug(_("Looking for direct download link...")) direct_link = self.getDirectLink(self.pyfile.url) if direct_link: return direct_link @@ -300,12 +300,12 @@ class SimpleHoster(Hoster): self.getFileInfo() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handled as premium download") + self.logDebug("Handle as premium download") self.handlePremium() elif premium_only: self.fail(_("Link require a premium account to be handled")) else: - self.logDebug("Handled as free download") + self.logDebug("Handle as free download") self.handleFree() @@ -318,7 +318,7 @@ class SimpleHoster(Hoster): if parseFileInfo(self, url, html)[2] is not 2: try: - return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1).strip() #@TODO: Remove .strip() in 0.4.10 + return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1).strip().lower() #@TODO: Remove .strip().lower() in 0.4.10 except: pass -- cgit v1.2.3 From 81d05814e4dc4f403b965dbfb0c713c0a6513355 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 7 Nov 2014 02:34:01 +0100 Subject: [XFSHoster] Fix a really silly typo --- module/plugins/internal/XFSHoster.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index b380c5f69..94c5d9595 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.12" + __version__ = "0.13" __pattern__ = r'^unmatchable$' @@ -75,12 +75,11 @@ class XFSHoster(SimpleHoster): self.errmsg = None self.passwords = self.getPassword().splitlines() - # MultiHoster check - if self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] - and re.match(self.__pattern__, self.pyfile.url) is None: + if (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] + and re.match(self.__pattern__, self.pyfile.url) is None): self.logInfo(_("Multi hoster detected")) if self.premium: - self.logDebug(_("Looking for leech download link...")) + self.logDebug(_("Looking for download link...")) self.handleOverriden() else: self.fail(_("Only premium users can use url leech feature")) @@ -182,7 +181,7 @@ class XFSHoster(SimpleHoster): if 'location' in header: #: Direct download link self.download(header['location'], ref=True, cookies=True, disposition=True) else: - self.error(_("No leech download link found")) + self.fail(_("No download link found")) def checkErrors(self): -- cgit v1.2.3 From 58c9c8d78417ec6ccac5b6b117ea540033a09574 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 7 Nov 2014 17:08:21 +0100 Subject: Fix download link formatting in some plugins --- module/plugins/internal/SimpleHoster.py | 4 +-- module/plugins/internal/XFSHoster.py | 57 +++++++++++++++++++-------------- 2 files changed, 35 insertions(+), 26 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 87a348367..6a8a6ff6f 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -200,7 +200,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.52" + __version__ = "0.53" __pattern__ = r'^unmatchable$' @@ -318,7 +318,7 @@ class SimpleHoster(Hoster): if parseFileInfo(self, url, html)[2] is not 2: try: - return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1).strip().lower() #@TODO: Remove .strip().lower() in 0.4.10 + return re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I).group(1).rstrip() #@TODO: Remove .rstrip() in 0.4.10 except: pass diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 94c5d9595..ecf1f9f97 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -3,20 +3,20 @@ import re from random import random +from time import sleep from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME from module.plugins.hoster.UnrestrictLi import secondsToMidnight from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia -from module.plugins.internal.SimpleHoster import create_getInfo, replace_patterns, set_cookies, SimpleHoster -from module.plugins.Plugin import Fail +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.13" + __version__ = "0.14" __pattern__ = r'^unmatchable$' @@ -96,11 +96,14 @@ class XFSHoster(SimpleHoster): self.download(link, ref=True, cookies=True, disposition=True) - elif self.errmsg and "captcha" in self.errmsg: - self.error(_("No valid captcha code entered")) + elif self.errmsg: + if 'captcha' in self.errmsg: + self.fail(_("No valid captcha code entered")) + else: + self.fail(self.errmsg) else: - self.error(_("Download link not found")) + self.fail(_("Download link not found")) def handlePremium(self): @@ -111,8 +114,16 @@ class XFSHoster(SimpleHoster): for i in xrange(1, 5): self.logDebug("Getting download link: #%d" % i) + self.checkErrors() + + m = re.search(self.LINK_PATTERN, self.html, re.S) + if m: + break + data = self.getPostParameters() + # sleep(10) + self.req.http.c.setopt(FOLLOWLOCATION, 0) self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True) @@ -120,15 +131,19 @@ class XFSHoster(SimpleHoster): self.req.http.c.setopt(FOLLOWLOCATION, 1) - m = re.search(r"Location\s*:\s*(.+)", self.header, re.I) + m = re.search(r'Location\s*:\s*(.+)', self.header, re.I) if m: break m = re.search(self.LINK_PATTERN, self.html, re.S) if m: break + else: + return - return m.group(1) + self.errmsg = None + + return m.group(1).strip() #@TODO: Remove .strip() in 0.4.10 def handleOverriden(self): @@ -158,13 +173,13 @@ class XFSHoster(SimpleHoster): stmsg = inputs['st'] - if stmsg == "OK": + if stmsg == 'OK': self.html = self.load(action, post=inputs) - elif "Can not leech file" in stmsg: + elif 'Can not leech file' in stmsg: self.retry(20, 3 * 60, _("Can not leech file")) - elif "all Leech traffic today" in stmsg: + elif 'today' in stmsg: self.retry(wait_time=secondsToMidnight(gmt=2), reason=_("You've used all Leech traffic today")) else: @@ -175,19 +190,21 @@ class XFSHoster(SimpleHoster): if m is None: self.error(_("OVR_LINK_PATTERN not found")) - self.pyfile.url = m.group(1) - - header = self.load(self.pyfile.url, just_header=True, decode=True) + header = self.load(m.group(1).strip(), just_header=True, decode=True) #@TODO: Remove .strip() in 0.4.10 if 'location' in header: #: Direct download link self.download(header['location'], ref=True, cookies=True, disposition=True) else: - self.fail(_("No download link found")) + self.fail(_("Download link not found")) def checkErrors(self): m = re.search(self.ERROR_PATTERN, self.html) - if m: + + if m is None: + self.errmsg = None + else: self.errmsg = m.group(1) + self.logWarning(re.sub(r"<.*?>", " ", self.errmsg)) if 'wait' in self.errmsg: @@ -223,16 +240,12 @@ class XFSHoster(SimpleHoster): else: self.fail(self.errmsg) - else: - self.errmsg = None return self.errmsg def getPostParameters(self): for _i in xrange(3): - self.checkErrors() - if hasattr(self, "FORM_PATTERN"): action, inputs = self.parseHtmlForm(self.FORM_PATTERN) else: @@ -268,9 +281,7 @@ class XFSHoster(SimpleHoster): if wait_time: self.wait() - self.errmsg = None return inputs - else: inputs['referer'] = self.pyfile.url @@ -284,8 +295,6 @@ class XFSHoster(SimpleHoster): del inputs['method_premium'] self.html = self.load(self.pyfile.url, post=inputs, ref=True) - self.errmsg = None - else: self.error(_("FORM: %s") % (inputs['op'] if 'op' in inputs else _("UNKNOWN"))) -- cgit v1.2.3 From 84b12612e3498d8e3543b2f54bbe3b848ee83460 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 8 Nov 2014 01:49:36 +0100 Subject: [XFSHoster] Improve ERROR_PATTERN --- module/plugins/internal/XFSHoster.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index ecf1f9f97..84c8dfa91 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.14" + __version__ = "0.15" __pattern__ = r'^unmatchable$' @@ -36,7 +36,7 @@ class XFSHoster(SimpleHoster): INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' NAME_PATTERN = r'[\d\.\,]+) ?(?P[\w^_]+)?\)' + SIZE_PATTERN = r'You have requested .*\((?P[\d.,]+) ?(?P[\w^_]+)?\)' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' @@ -51,7 +51,7 @@ class XFSHoster(SimpleHoster): RECAPTCHA_PATTERN = None SOLVEMEDIA_PATTERN = None - ERROR_PATTERN = r'(?:class=["\']err["\'][^>]*>|<[Cc]enter>)(.+?)(?:["\']|]*>|<[Cc]enter>)(.+?)(?:["\']|\(ERROR:(.+?)\)' def setup(self): @@ -165,9 +165,14 @@ class XFSHoster(SimpleHoster): self.html = self.load(action, post=inputs) + self.checkErrors() + action, inputs = self.parseHtmlForm('F1') if not inputs: - self.error(_("TEXTAREA F1 not found")) + if self.errmsg: + self.retry(reason=self.errmsg) + else: + self.error(_("TEXTAREA F1 not found")) self.logDebug(inputs) @@ -239,7 +244,7 @@ class XFSHoster(SimpleHoster): self.fail(_("File too large for free download")) else: - self.fail(self.errmsg) + self.retry(wait_time=60, reason=self.errmsg) return self.errmsg -- cgit v1.2.3 From 59f72bfc5ed721c80c821bd0ca1bc8daf0d49880 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 9 Nov 2014 03:12:41 +0100 Subject: Code cosmetics --- module/plugins/internal/CaptchaService.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 2143e23df..38775f519 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -91,7 +91,7 @@ class ReCaptcha(CaptchaService): self.plugin.fail(errmsg) raise TypeError(errmsg) - js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}, cookies=True) + js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}) try: challenge = re.search("challenge : '(.+?)',", js).group(1) @@ -153,7 +153,7 @@ class AdsCaptcha(CaptchaService): CaptchaId, PublicKey = key - js = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}, cookies=True) + js = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}) try: challenge = re.search("challenge: '(.+?)',", js).group(1) @@ -192,7 +192,7 @@ class SolveMedia(CaptchaService): self.plugin.fail(errmsg) raise TypeError(errmsg) - html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript", get={'k': key}, cookies=True) + html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript", get={'k': key}) try: challenge = re.search(r'', html).group(1) -- cgit v1.2.3 From c9e31d875d32de31e54959b82bc35eff2b3e0f3f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 10 Nov 2014 00:19:51 +0100 Subject: Code cosmetics --- module/plugins/internal/XFSAccount.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 8a2242018..ba2c00ec9 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -49,9 +49,9 @@ class XFSAccount(Account): def loadAccountInfo(self, user, req): - validuntil = None + validuntil = None trafficleft = None - premium = None + premium = None html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) @@ -64,8 +64,10 @@ class XFSAccount(Account): try: validuntil = mktime(strptime(expiredate, "%d %B %Y")) + except Exception, e: - self.logError(str(e)) + self.logError(e) + else: if validuntil > mktime(gmtime()): premium = True @@ -94,7 +96,7 @@ class XFSAccount(Account): trafficleft = self.parseTraffic(size + unit) except Exception, e: - self.logError(str(e)) + self.logError(e) return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} -- cgit v1.2.3 From 395ac7930ef4e1598186a7a0636d3f245b7e5134 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 11 Nov 2014 00:36:45 +0100 Subject: Fix url to name conversion in api and simple plugins --- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 69cbb7051..0e43496c4 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -129,7 +129,7 @@ class SimpleCrypter(Crypter): if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) + self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split("/")[-1] if status is 1: self.offline() diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6a8a6ff6f..6726d4fa5 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -329,7 +329,7 @@ class SimpleHoster(Hoster): if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.info['name'] = html_unescape(urlparse(url).path.split("/")[-1]) + self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split("/")[-1] if status is 1: self.offline() -- cgit v1.2.3 From 71c9121e0479db4ac1cb34ac4b4881e9b291772c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 13 Nov 2014 01:01:41 +0100 Subject: Spare code cosmetics --- module/plugins/internal/XFSHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 84c8dfa91..f54bb5e45 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -161,7 +161,7 @@ class XFSHoster(SimpleHoster): self.logDebug(action, inputs) - self.req.http.c.setopt(LOW_SPEED_TIME, 600) #: wait for file to upload to easybytez.com + self.req.setOption("timeout", 600) #: wait for file to upload to easybytez.com self.html = self.load(action, post=inputs) -- cgit v1.2.3 From d7d2b0c56e16866313b7f746f94569f085acef01 Mon Sep 17 00:00:00 2001 From: Radek Senfeld Date: Fri, 14 Nov 2014 01:47:39 +0100 Subject: [UlozTo] Fixed hoster and account --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6726d4fa5..1f7753003 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -324,7 +324,7 @@ class SimpleHoster(Hoster): def getFileInfo(self): - name, size, status, url = parseFileInfo(self) + name, size, status, url = parseFileInfo(self, html=self.html) if name and name != url: self.pyfile.name = name -- cgit v1.2.3 From 0521acfc4e62096d8f78e6f5d1966835c7acbcef Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 14 Nov 2014 01:53:24 +0100 Subject: [SimpleCrypter][SimpleHoster] Temp workaround --- module/plugins/internal/SimpleCrypter.py | 5 ++--- module/plugins/internal/SimpleHoster.py | 9 +++------ 2 files changed, 5 insertions(+), 9 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 0e43496c4..01d1895a6 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -13,7 +13,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.28" + __version__ = "0.29" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -96,8 +96,7 @@ class SimpleCrypter(Crypter): self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - if self.html is None: - self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) if isinstance(self.TEXT_ENCODING, basestring): self.html = unicode(self.html, self.TEXT_ENCODING) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1f7753003..2562f0af3 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -200,7 +200,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.53" + __version__ = "0.54" __pattern__ = r'^unmatchable$' @@ -274,11 +274,8 @@ class SimpleHoster(Hoster): return direct_link else: self.logDebug(_("No direct download link found")) - self.html = None - self.info = {} - if self.html is None: - self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) if isinstance(self.TEXT_ENCODING, basestring): self.html = unicode(self.html, self.TEXT_ENCODING) @@ -296,7 +293,7 @@ class SimpleHoster(Hoster): else: premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only and not self.info: #: Usually premium only pages doesn't show any file information + if not premium_only: #: Usually premium only pages doesn't show any file information self.getFileInfo() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): -- cgit v1.2.3 From 7d8d0991e4d9848325692f8bbb81df00afe4e065 Mon Sep 17 00:00:00 2001 From: Thierry Merle Date: Sat, 15 Nov 2014 07:14:06 +0100 Subject: [SimpleHoster][OneFichierCom] Fix issue #836 ('status': 3) --- module/plugins/internal/SimpleHoster.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 2562f0af3..18df2de92 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -170,10 +170,9 @@ def create_getInfo(plugin): def getInfo(urls): for url in urls: if hasattr(plugin, "COOKIES") and isinstance(plugin.COOKIES, list): - cj = CookieJar(plugin.__name__) - set_cookies(cj, plugin.COOKIES) + set_cookies(plugin.req.cj, plugin.COOKIES) else: - cj = None + plugin.req.cj = None if hasattr(plugin, "URL_REPLACEMENTS"): url = replace_patterns(url, plugin.URL_REPLACEMENTS) @@ -182,11 +181,11 @@ def create_getInfo(plugin): url = replace_patterns(url, plugin.FILE_URL_REPLACEMENTS) if hasattr(plugin, "TEXT_ENCODING"): - html = getURL(url, cookies=bool(cj), decode=not plugin.TEXT_ENCODING) + html = plugin.load(url, decode=not plugin.TEXT_ENCODING, cookies=bool(plugin.COOKIES)) if isinstance(plugin.TEXT_ENCODING, basestring): html = unicode(html, plugin.TEXT_ENCODING) else: - html = getURL(url, cookies=bool(cj), decode=True) + html = plugin.load(url, decode=True, cookies=bool(plugin.COOKIES)) yield parseFileInfo(plugin, url, html) @@ -200,7 +199,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.54" + __version__ = "0.55" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From c370ee8ed94c016d3e4e30b1917bc34e9510be0b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 20 Nov 2014 23:00:45 +0100 Subject: [XFSHoster] Fix StreamcloudEu (thx seoester) --- module/plugins/internal/XFSHoster.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index f54bb5e45..43f6cab83 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.15" + __version__ = "0.16" __pattern__ = r'^unmatchable$' @@ -41,7 +41,7 @@ class XFSHoster(SimpleHoster): OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' - WAIT_PATTERN = r'.*?>(\d+)' + WAIT_PATTERN = r'.*?>(\d+)|id="countdown" value=".*?(\d+).*?"' OVR_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' LINK_PATTERN = None #: final download url pattern @@ -122,8 +122,6 @@ class XFSHoster(SimpleHoster): data = self.getPostParameters() - # sleep(10) - self.req.http.c.setopt(FOLLOWLOCATION, 0) self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True) @@ -266,7 +264,7 @@ class XFSHoster(SimpleHoster): self.logDebug(inputs) - if 'op' in inputs and inputs['op'] in ("download2", "download3"): + if 'op' in inputs and inputs['op'] in ("download1", "download2", "download3"): if "password" in inputs: if self.passwords: inputs['password'] = self.passwords.pop(0) -- cgit v1.2.3 From 64afb5db7af0acdc74011b4e1a95ea83bd68bf39 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 06:30:16 +0100 Subject: [SimpleHoster] Updated --- module/plugins/internal/SimpleHoster.py | 319 +++++++++++++++++--------------- 1 file changed, 166 insertions(+), 153 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 18df2de92..e052a79ae 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -7,6 +7,7 @@ from urlparse import urlparse from pycurl import FOLLOWLOCATION +from module.PyFile import statusMap from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster @@ -76,130 +77,50 @@ def parseHtmlForm(attr_str, html, input_names=None): continue elif hasattr(val, "search") and re.match(val, inputs[key]): continue - break # attibute value does not match + break #: attibute value does not match else: - break # attibute name does not match + break #: attibute name does not match else: - return action, inputs # passed attribute check + return action, inputs #: passed attribute check else: # no attribute check return action, inputs - return {}, None # no matching form found + return {}, None #: no matching form found -def parseFileInfo(self, url="", html=""): - if not url and hasattr(self, "pyfile"): - url = self.pyfile.url +def parseFileInfo(plugin, url="", html=""): + info = plugin.getInfo(url, html) + return info['name'], info['size'], info['status'], info['url'] - info = {'name': url, 'size': 0, 'status': 3} - if not html: - if url: - return create_getInfo(self)([url]).next() - - elif hasattr(self, "req") and self.req.http.code == '404': - info['status'] = 1 - - elif hasattr(self, "html"): - html = self.html - - if html: - if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html): - info['status'] = 1 - - elif hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html): #@TODO: Remove in 0.4.10 - info['status'] = 1 - - elif hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, html): - info['status'] = 6 - - else: - online = False - try: - info.update(re.match(self.__pattern__, url).groupdict()) - except: - pass - - for pattern in ("INFO_PATTERN", "NAME_PATTERN", "SIZE_PATTERN", - "FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): #@TODO: Remove in 0.4.10 - try: - info.update(re.search(getattr(self, pattern), html).groupdict()) - online = True - except AttributeError: - continue - - if online: - # File online, return name and size - info['status'] = 2 +def create_getInfo(plugin): + return lambda urls: list(plugin.parseInfo(urls)) - if 'N' in info: - info['name'] = replace_patterns(info['N'].strip(), - self.FILE_NAME_REPLACEMENTS if hasattr(self, "FILE_NAME_REPLACEMENTS") else self.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 - if 'S' in info: - size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], - self.FILE_SIZE_REPLACEMENTS if hasattr(self, "FILE_SIZE_REPLACEMENTS") else self.SIZE_REPLACEMENTS) #@TODO: Remove FILE_SIZE_REPLACEMENTS check in 0.4.10 - info['size'] = parseFileSize(size) +def timestamp(): + return int(time() * 1000) - elif isinstance(info['size'], basestring): - unit = info['units'] if 'units' in info else None - info['size'] = parseFileSize(info['size'], unit) - if hasattr(self, "html") and self.html is None: - self.html = html +#@TODO: Move to hoster class in 0.4.10 +def _getDirectLink(self, url): + self.req.http.c.setopt(FOLLOWLOCATION, 0) - if hasattr(self, "info"): - try: - self.logDebug(_("File info (before update): %s") % self.info) - except: - pass + html = self.load(url, ref=True, decode=True) - self.info.update(info) + self.req.http.c.setopt(FOLLOWLOCATION, 1) + if self.getInfo(url, html)['status'] is not 2: try: - self.logDebug(_("File info (after update): %s") % self.info) + return re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I).group(1).rstrip() #@TODO: Remove .rstrip() in 0.4.10 except: pass - return info['name'], info['size'], info['status'], url - - -def create_getInfo(plugin): - - def getInfo(urls): - for url in urls: - if hasattr(plugin, "COOKIES") and isinstance(plugin.COOKIES, list): - set_cookies(plugin.req.cj, plugin.COOKIES) - else: - plugin.req.cj = None - - if hasattr(plugin, "URL_REPLACEMENTS"): - url = replace_patterns(url, plugin.URL_REPLACEMENTS) - - elif hasattr(plugin, "FILE_URL_REPLACEMENTS"): #@TODO: Remove in 0.4.10 - url = replace_patterns(url, plugin.FILE_URL_REPLACEMENTS) - - if hasattr(plugin, "TEXT_ENCODING"): - html = plugin.load(url, decode=not plugin.TEXT_ENCODING, cookies=bool(plugin.COOKIES)) - if isinstance(plugin.TEXT_ENCODING, basestring): - html = unicode(html, plugin.TEXT_ENCODING) - else: - html = plugin.load(url, decode=True, cookies=bool(plugin.COOKIES)) - - yield parseFileInfo(plugin, url, html) - - return getInfo - - -def timestamp(): - return int(time() * 1000) - class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.55" + __version__ = "0.56" __pattern__ = r'^unmatchable$' @@ -245,12 +166,79 @@ class SimpleHoster(Hoster): URL_REPLACEMENTS = [] TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] + COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + CHECK_DIRECT_LINK = None #: Set to None to set True if self.account else False + + + @classmethod + def parseInfo(cls, urls): + for url in urls: + url = replace_patterns(url, cls.FILE_URL_REPLACEMENTS if hasattr(cls, "FILE_URL_REPLACEMENTS") else cls.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 + yield cls.getInfo(cls, url) + + + @classmethod + def getInfo(cls, url="", html=""): + info = {'name': url or _("Unknown"), 'size': 0, 'status': 3, 'url': url} + + if not html: + if url: + html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING) + if isinstance(cls.TEXT_ENCODING, basestring): + html = unicode(html, cls.TEXT_ENCODING) + else: + return info + + online = False + + if hasattr(cls, "OFFLINE_PATTERN") and re.search(cls.OFFLINE_PATTERN, html): + info['status'] = 1 + + elif hasattr(cls, "FILE_OFFLINE_PATTERN") and re.search(cls.FILE_OFFLINE_PATTERN, html): #@TODO: Remove in 0.4.10 + info['status'] = 1 + + elif hasattr(cls, "TEMP_OFFLINE_PATTERN") and re.search(cls.TEMP_OFFLINE_PATTERN, html): + info['status'] = 6 + + else: + try: + info.update(re.match(cls.__pattern__, url).groupdict()) + except: + pass + + for pattern in ("INFO_PATTERN", "NAME_PATTERN", "SIZE_PATTERN", + "FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): #@TODO: Remove in 0.4.10 + try: + attr = getattr(cls, pattern) + info.update(re.search(attr, html).groupdict()) + except AttributeError: + continue + else: + online = True + + if online: + info['status'] = 2 + + if 'N' in info: + info['name'] = replace_patterns(info['N'].strip(), + cls.FILE_NAME_REPLACEMENTS if hasattr(cls, "FILE_NAME_REPLACEMENTS") else cls.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 + + if 'S' in info: + size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], + cls.FILE_SIZE_REPLACEMENTS if hasattr(cls, "FILE_SIZE_REPLACEMENTS") else cls.SIZE_REPLACEMENTS) #@TODO: Remove FILE_SIZE_REPLACEMENTS check in 0.4.10 + info['size'] = parseFileSize(size) + + elif isinstance(info['size'], basestring): + unit = info['units'] if 'units' in info else None + info['size'] = parseFileSize(info['size'], unit) + + return info def init(self): - self.info = {} + self.info = {} #@TODO: Remove in 0.4.10 + self.link = None #@TODO: Move to hoster class in 0.4.10 def setup(self): @@ -258,91 +246,118 @@ class SimpleHoster(Hoster): def prepare(self): - if isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) + if self.CHECK_DIRECT_LINK is None: + self.CHECK_DIRECT_LINK = bool(self.account) self.req.setOption("timeout", 120) + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) + self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 - if self.premium: - self.logDebug(_("Looking for direct download link...")) - direct_link = self.getDirectLink(self.pyfile.url) - if direct_link: - return direct_link - else: - self.logDebug(_("No direct download link found")) - self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + def preload(self): + self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING) if isinstance(self.TEXT_ENCODING, basestring): self.html = unicode(self.html, self.TEXT_ENCODING) def process(self, pyfile): - direct_link = self.prepare() + self.prepare() - if isinstance(direct_link, basestring): - self.logInfo(_("Direct download link detected")) - self.download(direct_link, ref=True, cookies=True, disposition=True) + if self.CHECK_DIRECT_LINK: + self.logDebug("Looking for direct download link...") + self.handleDirect() - elif self.html is None: - self.fail(_("No html retrieved")) + if not self.link: + self.preload() + + #@TODO: Remove in 0.4.10 + if self.html is None: + self.fail(_("No html retrieved")) + + info = self.getInfo(pyfile.url, self.html) + self._updateInfo(info) + + self.checkNameSize() - else: premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) if not premium_only: #: Usually premium only pages doesn't show any file information - self.getFileInfo() + self.checkStatus() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handle as premium download") + self.logDebug("Handled as premium download") self.handlePremium() + elif premium_only: self.fail(_("Link require a premium account to be handled")) + else: - self.logDebug("Handle as free download") + self.logDebug("Handled as free download") self.handleFree() + if self.link: + self.download(self.link) - def getDirectLink(self, url): - self.req.http.c.setopt(FOLLOWLOCATION, 0) - html = self.load(url, ref=True, decode=True) + def checkStatus(self): + status = self.info['status'] - self.req.http.c.setopt(FOLLOWLOCATION, 1) + if status is 1: + self.offline() - if parseFileInfo(self, url, html)[2] is not 2: - try: - return re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I).group(1).rstrip() #@TODO: Remove .rstrip() in 0.4.10 - except: - pass + elif status is 6: + self.tempOffline() + + elif status is not 2: + self.error(_("File status: %s") % filter(lambda key, val: val == status, statusMap.iteritems())[0], + _("File info: %s") % self.info) - def getFileInfo(self): - name, size, status, url = parseFileInfo(self, html=self.html) + def checkNameSize(self): + name = self.info['name'] + size = self.info['size'] + url = self.info['url'] if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split("/")[-1] + self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split('/')[-1] - if status is 1: - self.offline() + if size > 0: + self.pyfile.size = size + else: + self.logError(_("File size not found")) - elif status is 6: - self.tempOffline() + self.logDebug("File name: %s" % self.pyfile.name, "File size: %s" % self.pyfile.size or _("Unknown")) - elif status is not 2: - self.error(_("File info: %s") % self.info) - if size: - self.pyfile.size = size - else: - self.logError(_("File size not parsed")) + def checkInfo(self): + self._updateInfo(self.getInfo(self.pyfile.url, self.html or "")) + self.checkNameSize() + self.checkStatus() + + + def _updateInfo(self, info) + self.logDebug(_("File info (previous): %s") % self.info) + self.info.update(info) + self.logDebug(_("File info (current): %s") % self.info) + - self.logDebug("FILE NAME: %s" % self.pyfile.name, "FILE SIZE: %d" % self.pyfile.size or _("Unknown")) - return self.info + def handleDirect(self): + self.link = _getDirectLink(self, self.pyfile.url) + + if self.link: + self.logInfo(_("Direct download link detected")) + + self._updateInfo(self.getInfo(self.pyfile.url)) + self.checkNameSize() + + else: + self.logDebug(_("Direct download link not found")) def handleFree(self): @@ -354,11 +369,10 @@ class SimpleHoster(Hoster): if m is None: self.error(_("Free download link not found")) - link = m.group(1) + self.link = m.group(1) + except Exception, e: self.fail(str(e)) - else: - self.download(link, ref=True, cookies=True, disposition=True) def handlePremium(self): @@ -370,19 +384,18 @@ class SimpleHoster(Hoster): if m is None: self.error(_("Premium download link not found")) - link = m.group(1) + self.link = m.group(1) + except Exception, e: self.fail(str(e)) - else: - self.download(link, ref=True, cookies=True, disposition=True) def longWait(self, wait_time=None, max_tries=3): if wait_time and isinstance(wait_time, (int, long, float)): - time_str = "%dh %dm" % divmod(wait_time / 60, 60) + time_str = "%dh %dm" % divmod(wait_time / 60, 60) else: wait_time = 900 - time_str = _("(unknown time)") + time_str = _("(unknown time)") max_tries = 100 self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str) -- cgit v1.2.3 From 353f0b64180364cd2d04a3fac0eba5358203eab4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 07:09:49 +0100 Subject: [SimpleCrypter] Updated --- module/plugins/internal/SimpleCrypter.py | 50 +++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 14 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 01d1895a6..d1938ec66 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -13,7 +13,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.29" + __version__ = "0.30" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -71,7 +71,8 @@ class SimpleCrypter(Crypter): #@TODO: remove in 0.4.10 def init(self): - self.info = {} + self.info = {} #@TODO: Remove in 0.4.10 + self.links = [] account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") account = self.core.accountManager.getAccountPlugin(account_name) @@ -91,12 +92,16 @@ class SimpleCrypter(Crypter): if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) + self.req.setOption("timeout", 120) + if isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + + def preload(self): + self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING) if isinstance(self.TEXT_ENCODING, basestring): self.html = unicode(self.html, self.TEXT_ENCODING) @@ -105,11 +110,12 @@ class SimpleCrypter(Crypter): def decrypt(self, pyfile): self.prepare() + self.preload() + if self.html is None: self.fail(_("No html retrieved")) - if not self.info: - self.getFileInfo() + self.checkInfo() self.links = self.getLinks() @@ -122,13 +128,8 @@ class SimpleCrypter(Crypter): self.packages = [(self.info['name'], self.links, self.info['folder'])] - def getFileInfo(self): - name, size, status, url = parseFileInfo(self) - - if name and name != url: - self.pyfile.name = name - else: - self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split("/")[-1] + def checkStatus(self): + status = self.info['status'] if status is 1: self.offline() @@ -136,10 +137,31 @@ class SimpleCrypter(Crypter): elif status is 6: self.tempOffline() + + def checkName(self): + name = self.info['name'] + url = self.info['url'] + + if name and name != url: + self.pyfile.name = name + else: + self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split('/')[-1] + self.info['folder'] = self.pyfile.name - self.logDebug("FILE NAME: %s" % self.pyfile.name) - return self.info + self.logDebug("File name: %s" % self.pyfile.name) + + + def checkInfo(self): + self.logDebug(_("File info (previous): %s") % self.info) + + info = parseFileInfo(self.pyfile.url, self.html or "") + self.info.update(info) + + self.logDebug(_("File info (current): %s") % self.info) + + self.checkName() + self.checkStatus() def getLinks(self): -- cgit v1.2.3 From f9f18d96663a1d50dee9c8bc45766ffd1af9fba8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 07:10:08 +0100 Subject: [XFSHoster] Updated --- module/plugins/internal/XFSHoster.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 43f6cab83..ea5889190 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.16" + __version__ = "0.17" __pattern__ = r'^unmatchable$' @@ -71,6 +71,9 @@ class XFSHoster(SimpleHoster): pattern = r'(https?://(www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]' self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.') + if self.CHECK_DIRECT_LINK is None: + self.CHECK_DIRECT_LINK = bool(self.premium) + self.captcha = None self.errmsg = None self.passwords = self.getPassword().splitlines() -- cgit v1.2.3 From 8bcb70d042bf93fefe6f6b3737ce0bee396bcf75 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 07:10:34 +0100 Subject: [SimpleHoster] CONTENT_DISPOSITION feature --- module/plugins/internal/SimpleHoster.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e052a79ae..9cf8eb541 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -165,10 +165,11 @@ class SimpleHoster(Hoster): SIZE_REPLACEMENTS = [] URL_REPLACEMENTS = [] - TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct + TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - CHECK_DIRECT_LINK = None #: Set to None to set True if self.account else False + CHECK_DIRECT_LINK = None #: Set to None self-set to True if self.account else to False + CONTENT_DISPOSITION = False #: Set to True to replace file name with content-disposition value in http header @classmethod @@ -237,8 +238,8 @@ class SimpleHoster(Hoster): def init(self): - self.info = {} #@TODO: Remove in 0.4.10 - self.link = None #@TODO: Move to hoster class in 0.4.10 + self.info = {} #@TODO: Remove in 0.4.10 + self.link = "" #@TODO: Move to hoster class in 0.4.10 def setup(self): @@ -300,7 +301,7 @@ class SimpleHoster(Hoster): self.handleFree() if self.link: - self.download(self.link) + self.download(self.link, disposition=self.CONTENT_DISPOSITION) def checkStatus(self): -- cgit v1.2.3 From acddd89ad9e0ce9107ac9a99443cfbecc58a1ff9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 07:46:34 +0100 Subject: [SimpleCrypter][SimpleHoster] Fix info attr init --- module/plugins/internal/SimpleCrypter.py | 6 +++--- module/plugins/internal/SimpleHoster.py | 8 +++----- 2 files changed, 6 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index d1938ec66..75c687c16 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -71,9 +71,6 @@ class SimpleCrypter(Crypter): #@TODO: remove in 0.4.10 def init(self): - self.info = {} #@TODO: Remove in 0.4.10 - self.links = [] - account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") account = self.core.accountManager.getAccountPlugin(account_name) @@ -92,6 +89,9 @@ class SimpleCrypter(Crypter): if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) + self.info = {} + self.links = [] + self.req.setOption("timeout", 120) if isinstance(self.COOKIES, list): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 9cf8eb541..7ab562779 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -237,16 +237,14 @@ class SimpleHoster(Hoster): return info - def init(self): - self.info = {} #@TODO: Remove in 0.4.10 - self.link = "" #@TODO: Move to hoster class in 0.4.10 - - def setup(self): self.resumeDownload = self.multiDL = self.premium def prepare(self): + self.info = {} + self.link = "" #@TODO: Move to hoster class in 0.4.10 + if self.CHECK_DIRECT_LINK is None: self.CHECK_DIRECT_LINK = bool(self.account) -- cgit v1.2.3 From e9c00be0d343ad30bca51147160b5067d7f9c1a8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 18:18:13 +0100 Subject: [XFSHoster] Update info patterns --- module/plugins/internal/XFSHoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index ea5889190..de9fbc831 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.17" + __version__ = "0.18" __pattern__ = r'^unmatchable$' @@ -35,8 +35,8 @@ class XFSHoster(SimpleHoster): COOKIES = [(HOSTER_DOMAIN, "lang", "english")] INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' - NAME_PATTERN = r'[\d.,]+) ?(?P[\w^_]+)?\)' + NAME_PATTERN = r'(>Filename:|name="fname" value="|)(?P.+?)(\s*<|")' + SIZE_PATTERN = r'(>Size:|>File:.*>|)(?P[\d.,]+)\s*(?P[\w^_]+)' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' -- cgit v1.2.3 From 48d785908bafd2615189c6792cbfec36db7e97df Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 18:20:17 +0100 Subject: [SimpleCrypter] Extend SimpleHoster --- module/plugins/internal/SimpleCrypter.py | 47 +++++++------------------------- 1 file changed, 10 insertions(+), 37 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 75c687c16..8c7f0bef3 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -6,14 +6,14 @@ from urlparse import urlparse from module.plugins.Crypter import Crypter from module.plugins.Plugin import Fail -from module.plugins.internal.SimpleHoster import _error, _wait, parseFileInfo, replace_patterns, set_cookies -from module.utils import fixup, html_unescape +from module.plugins.internal.SimpleHoster import SimpleHoster, _error, _wait, replace_patterns, set_cookies +from module.utils import fixup -class SimpleCrypter(Crypter): +class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.30" + __version__ = "0.31" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -69,7 +69,7 @@ class SimpleCrypter(Crypter): LOGIN_PREMIUM = False - #@TODO: remove in 0.4.10 + #@TODO: Remove in 0.4.10 def init(self): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") account = self.core.accountManager.getAccountPlugin(account_name) @@ -100,13 +100,6 @@ class SimpleCrypter(Crypter): self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - def preload(self): - self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING) - - if isinstance(self.TEXT_ENCODING, basestring): - self.html = unicode(self.html, self.TEXT_ENCODING) - - def decrypt(self, pyfile): self.prepare() @@ -138,30 +131,19 @@ class SimpleCrypter(Crypter): self.tempOffline() - def checkName(self): + def checkNameSize(self): name = self.info['name'] url = self.info['url'] if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split('/')[-1] - - self.info['folder'] = self.pyfile.name - - self.logDebug("File name: %s" % self.pyfile.name) - + self.pyfile.name = self.info['name'] = urlparse(name).path.split('/')[-1] - def checkInfo(self): - self.logDebug(_("File info (previous): %s") % self.info) + folder = self.info['folder'] = self.pyfile.name - info = parseFileInfo(self.pyfile.url, self.html or "") - self.info.update(info) - - self.logDebug(_("File info (current): %s") % self.info) - - self.checkName() - self.checkStatus() + self.logDebug("File name: %s" % self.pyfile.name, + "File folder: %s" % folder) def getLinks(self): @@ -182,12 +164,3 @@ class SimpleCrypter(Crypter): for p in xrange(2, pages + 1): self.html = self.loadPage(p) self.links += self.getLinks() - - - #@TODO: Remove in 0.4.10 - def wait(self, seconds=0, reconnect=None): - return _wait(self, seconds, reconnect) - - - def error(self, reason="", type="parse"): - return _error(self, reason, type) -- cgit v1.2.3 From e08fb5d3361627ad866aed86be9597b32c8c50f9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 18:23:33 +0100 Subject: [SimpleHoster] Code cosmetics --- module/plugins/internal/SimpleHoster.py | 39 ++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 15 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 7ab562779..1de872261 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -7,12 +7,16 @@ from urlparse import urlparse from pycurl import FOLLOWLOCATION -from module.PyFile import statusMap +from module.PyFile import statusMap as _statusMap from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster from module.plugins.Plugin import Fail -from module.utils import fixup, html_unescape, parseFileSize +from module.utils import fixup, parseFileSize + + +#@TODO: Adapt and move to PyFile in 0.4.10 +statusMap = {v: k for k, v in _statusMap.iteritems()} #@TODO: Remove in 0.4.10 and redirect to self.error instead @@ -21,7 +25,7 @@ def _error(self, reason, type): type = "unknown" msg = _("%s error") % type.strip().capitalize() if type else _("Error") - msg += ": " + reason.strip() if reason else "" + msg += ": %s" % reason.strip() if reason else "" msg += _(" | Plugin may be out of date") raise Fail(msg) @@ -89,11 +93,13 @@ def parseHtmlForm(attr_str, html, input_names=None): return {}, None #: no matching form found +#: Deprecated def parseFileInfo(plugin, url="", html=""): info = plugin.getInfo(url, html) return info['name'], info['size'], info['status'], info['url'] +#@TODO: Remove in 0.4.10 def create_getInfo(plugin): return lambda urls: list(plugin.parseInfo(urls)) @@ -120,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.56" + __version__ = "0.57" __pattern__ = r'^unmatchable$' @@ -279,12 +285,13 @@ class SimpleHoster(Hoster): self.fail(_("No html retrieved")) info = self.getInfo(pyfile.url, self.html) - self._updateInfo(info) + premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) + self._updateInfo(info) self.checkNameSize() - premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only: #: Usually premium only pages doesn't show any file information + #: Usually premium only pages doesn't show any file information + if not premium_only: self.checkStatus() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): @@ -312,8 +319,9 @@ class SimpleHoster(Hoster): self.tempOffline() elif status is not 2: - self.error(_("File status: %s") % filter(lambda key, val: val == status, statusMap.iteritems())[0], - _("File info: %s") % self.info) + self.logDebug("File status: %s" % statusMap[status], + "File info: %s" % self.info) + self.error(_("No file info")) def checkNameSize(self): @@ -324,14 +332,15 @@ class SimpleHoster(Hoster): if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split('/')[-1] + self.pyfile.name = name = self.info['name'] = urlparse(name).path.split('/')[-1] if size > 0: self.pyfile.size = size else: - self.logError(_("File size not found")) + size = "Unknown" - self.logDebug("File name: %s" % self.pyfile.name, "File size: %s" % self.pyfile.size or _("Unknown")) + self.logDebug("File name: %s" % name, + "File size: %s" % size) def checkInfo(self): @@ -340,10 +349,10 @@ class SimpleHoster(Hoster): self.checkStatus() - def _updateInfo(self, info) - self.logDebug(_("File info (previous): %s") % self.info) + def _updateInfo(self, info): + self.logDebug(_("File info (before update): %s") % self.info) self.info.update(info) - self.logDebug(_("File info (current): %s") % self.info) + self.logDebug(_("File info (after update): %s") % self.info) def handleDirect(self): -- cgit v1.2.3 From 911ef70f5e39444d4b885b79ba825855b54e7606 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 18:59:47 +0100 Subject: [SimpleHoster] Fix create_getInfo --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1de872261..c5875a8da 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -101,7 +101,7 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 def create_getInfo(plugin): - return lambda urls: list(plugin.parseInfo(urls)) + return lambda urls: [info['name'], info['size'], info['status'], info['url'] for info in plugin.parseInfo(urls)] def timestamp(): -- cgit v1.2.3 From 70759ae9fd8561436bef08322d45ac41637c66d6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 20:01:24 +0100 Subject: [XFSHoster] Fix getPostParameters --- module/plugins/internal/XFSHoster.py | 102 +++++++++++++++++------------------ 1 file changed, 50 insertions(+), 52 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index de9fbc831..d2a771507 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.18" + __version__ = "0.19" __pattern__ = r'^unmatchable$' @@ -32,7 +32,9 @@ class XFSHoster(SimpleHoster): URL_REPLACEMENTS = [(r'/(?:embed-)?(\w{12}).*', r'/\1')] #: plus support embedded files - COOKIES = [(HOSTER_DOMAIN, "lang", "english")] + TEXT_ENCODING = False + COOKIES = [(HOSTER_DOMAIN, "lang", "english")] + CHECK_DIRECT_LINK = None INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' NAME_PATTERN = r'(>Filename:|name="fname" value="|)(?P.+?)(\s*<|")' @@ -128,19 +130,19 @@ class XFSHoster(SimpleHoster): self.req.http.c.setopt(FOLLOWLOCATION, 0) self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True) - self.header = self.req.http.header self.req.http.c.setopt(FOLLOWLOCATION, 1) - m = re.search(r'Location\s*:\s*(.+)', self.header, re.I) - if m: + m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) + if m and not "op=" in m.group(1): break m = re.search(self.LINK_PATTERN, self.html, re.S) if m: break else: - return + self.logError(data['op'] if 'op' in data else _("UNKNOWN")) + return "" self.errmsg = None @@ -197,6 +199,7 @@ class XFSHoster(SimpleHoster): self.error(_("OVR_LINK_PATTERN not found")) header = self.load(m.group(1).strip(), just_header=True, decode=True) #@TODO: Remove .strip() in 0.4.10 + if 'location' in header: #: Direct download link self.download(header['location'], ref=True, cookies=True, disposition=True) else: @@ -251,58 +254,53 @@ class XFSHoster(SimpleHoster): def getPostParameters(self): - for _i in xrange(3): - if hasattr(self, "FORM_PATTERN"): - action, inputs = self.parseHtmlForm(self.FORM_PATTERN) - else: - action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) + if hasattr(self, "FORM_PATTERN"): + action, inputs = self.parseHtmlForm(self.FORM_PATTERN) + else: + action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) + if not inputs: + action, inputs = self.parseHtmlForm('F1') if not inputs: - action, inputs = self.parseHtmlForm('F1') - if not inputs: - if self.errmsg: - self.retry(reason=self.errmsg) - else: - self.error(_("TEXTAREA F1 not found")) - - self.logDebug(inputs) - - if 'op' in inputs and inputs['op'] in ("download1", "download2", "download3"): - if "password" in inputs: - if self.passwords: - inputs['password'] = self.passwords.pop(0) - else: - self.fail(_("No or invalid passport")) - - if not self.premium: - m = re.search(self.WAIT_PATTERN, self.html) - if m: - wait_time = int(m.group(1)) - self.setWait(wait_time, False) - else: - wait_time = 0 - - self.captcha = self.handleCaptcha(inputs) - - if wait_time: - self.wait() - - return inputs - else: - inputs['referer'] = self.pyfile.url + if self.errmsg: + self.retry(reason=self.errmsg) + else: + self.error(_("TEXTAREA F1 not found")) - if self.premium: - inputs['method_premium'] = "Premium Download" - if 'method_free' in inputs: - del inputs['method_free'] + self.logDebug(inputs) + + if 'op' in inputs: + if "password" in inputs: + if self.passwords: + inputs['password'] = self.passwords.pop(0) + else: + self.fail(_("No or invalid passport")) + + if not self.premium: + m = re.search(self.WAIT_PATTERN, self.html) + if m: + wait_time = int(m.group(1)) + self.setWait(wait_time, False) else: - inputs['method_free'] = "Free Download" - if 'method_premium' in inputs: - del inputs['method_premium'] + wait_time = 0 + + self.captcha = self.handleCaptcha(inputs) - self.html = self.load(self.pyfile.url, post=inputs, ref=True) + if wait_time: + self.wait() else: - self.error(_("FORM: %s") % (inputs['op'] if 'op' in inputs else _("UNKNOWN"))) + inputs['referer'] = self.pyfile.url + + if self.premium: + inputs['method_premium'] = "Premium Download" + if 'method_free' in inputs: + del inputs['method_free'] + else: + inputs['method_free'] = "Free Download" + if 'method_premium' in inputs: + del inputs['method_premium'] + + return inputs def handleCaptcha(self, inputs): -- cgit v1.2.3 From 4dfd0fe2f76e67218158d76644a48f58a95bcf8d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 01:35:32 +0100 Subject: [MultiHoster] Fix __name__ --- module/plugins/internal/MultiHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index f0d3993a7..c86f9b286 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -7,7 +7,7 @@ from module.utils import remove_chars class MultiHoster(Hook): - __name__ = "AbtractExtractor" + __name__ = "MultiHoster" __type__ = "hook" __version__ = "0.19" -- cgit v1.2.3 From e6c7bc85b5a18bef90ea0f3a3c235d3849c28b89 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 01:37:33 +0100 Subject: [XFSHoster] Improvements --- module/plugins/internal/XFSHoster.py | 71 +++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 34 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index d2a771507..0c428c032 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.19" + __version__ = "0.20" __pattern__ = r'^unmatchable$' @@ -35,6 +35,7 @@ class XFSHoster(SimpleHoster): TEXT_ENCODING = False COOKIES = [(HOSTER_DOMAIN, "lang", "english")] CHECK_DIRECT_LINK = None + MULTI_HOSTER = False INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' NAME_PATTERN = r'(>Filename:|name="fname" value="|)(?P.+?)(\s*<|")' @@ -43,18 +44,18 @@ class XFSHoster(SimpleHoster): OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' - WAIT_PATTERN = r'.*?>(\d+)|id="countdown" value=".*?(\d+).*?"' + WAIT_PATTERN = r'.*?>(\d+)|id="countdown" value=".*?(\d+).*?"' + PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only' + ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter>|>Error\s*\s*(?:<.+?>)?)(.+?)(?:["\']|\(ERROR:(.+?)\)' OVR_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' LINK_PATTERN = None #: final download url pattern - CAPTCHA_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' + CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' CAPTCHA_DIV_PATTERN = r'>Enter code.*?(.+?)
' RECAPTCHA_PATTERN = None SOLVEMEDIA_PATTERN = None - ERROR_PATTERN = r'(?:class=["\']err["\'][^>]*>|<[Cc]enter>)(.+?)(?:["\']|\(ERROR:(.+?)\)' - def setup(self): self.chunkLimit = 1 @@ -73,23 +74,14 @@ class XFSHoster(SimpleHoster): pattern = r'(https?://(www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]' self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.') - if self.CHECK_DIRECT_LINK is None: - self.CHECK_DIRECT_LINK = bool(self.premium) - - self.captcha = None - self.errmsg = None + self.captcha = None + self.errmsg = None self.passwords = self.getPassword().splitlines() - if (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] - and re.match(self.__pattern__, self.pyfile.url) is None): - self.logInfo(_("Multi hoster detected")) - if self.premium: - self.logDebug(_("Looking for download link...")) - self.handleOverriden() - else: - self.fail(_("Only premium users can use url leech feature")) + super(XFSHoster, self).prepare() - return super(XFSHoster, self).prepare() + if self.CHECK_DIRECT_LINK is None: + self.directDL = bool(self.premium) def handleFree(self): @@ -134,8 +126,11 @@ class XFSHoster(SimpleHoster): self.req.http.c.setopt(FOLLOWLOCATION, 1) m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) - if m and not "op=" in m.group(1): - break + if m: + if 'down_direct' in data and data['down_direct']: + break + else: + self.error(_("Direct download link detected but not found")) m = re.search(self.LINK_PATTERN, self.html, re.S) if m: @@ -149,7 +144,7 @@ class XFSHoster(SimpleHoster): return m.group(1).strip() #@TODO: Remove .strip() in 0.4.10 - def handleOverriden(self): + def handleMulti(self): #only tested with easybytez.com self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) @@ -201,12 +196,17 @@ class XFSHoster(SimpleHoster): header = self.load(m.group(1).strip(), just_header=True, decode=True) #@TODO: Remove .strip() in 0.4.10 if 'location' in header: #: Direct download link - self.download(header['location'], ref=True, cookies=True, disposition=True) + self.link = header['location'] else: self.fail(_("Download link not found")) def checkErrors(self): + m = re.search(self.PREMIUM_ONLY_PATTERN, self.html) + if m: + self.info['error'] = "premium-only" + return + m = re.search(self.ERROR_PATTERN, self.html) if m is None: @@ -217,10 +217,13 @@ class XFSHoster(SimpleHoster): self.logWarning(re.sub(r"<.*?>", " ", self.errmsg)) if 'wait' in self.errmsg: - wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in - re.findall(r'(\d+)\s*(hour|minute|second)', self.errmsg)]) + wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec)', self.errmsg, re.I)]) self.wait(wait_time, True) + elif 'country' in self.errmsg: + self.fail(_("Downloads are disabled for your country")) + elif 'captcha' in self.errmsg: self.invalidCaptcha() @@ -250,6 +253,11 @@ class XFSHoster(SimpleHoster): else: self.retry(wait_time=60, reason=self.errmsg) + if self.errmsg: + self.info['error'] = self.errmsg + else: + self.info.pop('error', None) + return self.errmsg @@ -274,31 +282,26 @@ class XFSHoster(SimpleHoster): if self.passwords: inputs['password'] = self.passwords.pop(0) else: - self.fail(_("No or invalid passport")) + self.fail(_("Missing password")) if not self.premium: m = re.search(self.WAIT_PATTERN, self.html) if m: wait_time = int(m.group(1)) self.setWait(wait_time, False) - else: - wait_time = 0 self.captcha = self.handleCaptcha(inputs) - if wait_time: - self.wait() + self.wait() else: inputs['referer'] = self.pyfile.url if self.premium: inputs['method_premium'] = "Premium Download" - if 'method_free' in inputs: - del inputs['method_free'] + inputs.pop('method_free', None) else: inputs['method_free'] = "Free Download" - if 'method_premium' in inputs: - del inputs['method_premium'] + inputs.pop('method_premium', None) return inputs -- cgit v1.2.3 From d85b5694472c4bf3983729a077a76dd58da25ac4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 01:46:38 +0100 Subject: [SimpleHoster] Multi-hoster and error handling support + fixes (thx MaPePeR) --- module/plugins/internal/SimpleCrypter.py | 3 +- module/plugins/internal/SimpleHoster.py | 95 ++++++++++++++++++++++++++------ 2 files changed, 79 insertions(+), 19 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 8c7f0bef3..5d00daa3c 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -5,8 +5,7 @@ import re from urlparse import urlparse from module.plugins.Crypter import Crypter -from module.plugins.Plugin import Fail -from module.plugins.internal.SimpleHoster import SimpleHoster, _error, _wait, replace_patterns, set_cookies +from module.plugins.internal.SimpleHoster import SimpleHoster, replace_patterns, set_cookies from module.utils import fixup diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index c5875a8da..9b74c7cef 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -101,7 +101,7 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 def create_getInfo(plugin): - return lambda urls: [info['name'], info['size'], info['status'], info['url'] for info in plugin.parseInfo(urls)] + return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfo(urls)] def timestamp(): @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.57" + __version__ = "0.58" __pattern__ = r'^unmatchable$' @@ -138,7 +138,7 @@ class SimpleHoster(Hoster): """ - Following patterns should be defined by each hoster: + Info patterns should be defined by each hoster: INFO_PATTERN: (optional) Name and Size of the file example: INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' @@ -148,17 +148,26 @@ class SimpleHoster(Hoster): SIZE_PATTERN: (optional) Size that will be checked for the file example: SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' - OFFLINE_PATTERN: (optional) Checks if the file is yet available online + OFFLINE_PATTERN: (optional) Check if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN: (optional) Checks if the file is temporarily offline + TEMP_OFFLINE_PATTERN: (optional) Check if the file is temporarily offline example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' - PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account + + Error handling patterns are all optional: + + WAIT_PATTERN: (optional) Detect waiting time + example: WAIT_PATTERN = r'' + + PREMIUM_ONLY_PATTERN: (optional) Check if the file can be downloaded only with a premium account example: PREMIUM_ONLY_PATTERN = r'Premium account required' + ERROR_PATTERN: (optional) Detect any error preventing download + example: ERROR_PATTERN = r'' + - Instead overriding handleFree and handlePremium methods now you can define patterns for direct download: + Instead overriding handleFree and handlePremium methods you can define the following patterns for direct download: LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download example: LINK_FREE_PATTERN = r'
Date: Sat, 22 Nov 2014 05:15:25 +0100 Subject: [SimpleHoster] Fixup --- module/plugins/internal/SimpleHoster.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 9b74c7cef..a39531b82 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.58" + __version__ = "0.59" __pattern__ = r'^unmatchable$' @@ -192,7 +192,7 @@ class SimpleHoster(Hoster): def parseInfo(cls, urls): for url in urls: url = replace_patterns(url, cls.FILE_URL_REPLACEMENTS if hasattr(cls, "FILE_URL_REPLACEMENTS") else cls.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 - yield cls.getInfo(cls, url) + yield cls.getInfo(url) @classmethod @@ -272,7 +272,7 @@ class SimpleHoster(Hoster): and self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] and re.match(self.__pattern__, self.pyfile.url) is None): - self.logDebug("Multi hoster detected") + self.logInfo("Multi hoster detected") if self.account: self.multihost = True @@ -373,9 +373,9 @@ class SimpleHoster(Hoster): self.tempOffline() elif status is not 2: - self.logDebug("File status: %s" % statusMap[status], - "File info: %s" % self.info) - self.error(_("No file info")) + self.logInfo(_("File status: %s") % statusMap[status], + _("File info: %s") % self.info) + self.error(_("No file info retrieved")) def checkNameSize(self): @@ -406,6 +406,11 @@ class SimpleHoster(Hoster): self.checkStatus() + #: Deprecated + def getFileInfo(self): + return self.checkInfo() + + def _updateInfo(self, info): self.logDebug(_("File info (before update): %s") % self.info) self.info.update(info) -- cgit v1.2.3 From 9f10e758325f4ce80c479bc16149e677b9e1885f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 06:38:17 +0100 Subject: [XFSAccount][XFSHoster] Tiny pattern improvement --- module/plugins/internal/XFSAccount.py | 5 ++++- module/plugins/internal/XFSHoster.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index ba2c00ec9..627dc75c4 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.25" + __version__ = "0.26" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -97,6 +97,9 @@ class XFSAccount(Account): except Exception, e: self.logError(e) + else: + if premium: + trafficleft = -1 return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 0c428c032..a6a3f3260 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.20" + __version__ = "0.21" __pattern__ = r'^unmatchable$' @@ -38,7 +38,7 @@ class XFSHoster(SimpleHoster): MULTI_HOSTER = False INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' - NAME_PATTERN = r'(>Filename:|name="fname" value="|)(?P.+?)(\s*<|")' + NAME_PATTERN = r'(>Filename:|name="fname" value="||<[Tt]itle>.*?Download )(?P.+?)(\s*<|")' SIZE_PATTERN = r'(>Size:|>File:.*>|)(?P[\d.,]+)\s*(?P[\w^_]+)' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' -- cgit v1.2.3 From 561357c1e1fdb86a4ad89f146f8a1d1241d21a3b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 17:09:17 +0100 Subject: [XFSHoster] Improve ERROR_PATTERN + revert direct link detection in getDownloadLink --- module/plugins/internal/XFSHoster.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index a6a3f3260..d5bbf5c63 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.21" + __version__ = "0.22" __pattern__ = r'^unmatchable$' @@ -46,7 +46,7 @@ class XFSHoster(SimpleHoster): WAIT_PATTERN = r'.*?>(\d+)|id="countdown" value=".*?(\d+).*?"' PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only' - ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter>|>Error\s*\s*(?:<.+?>)?)(.+?)(?:["\']|\(ERROR:(.+?)\)' + ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter>|>Error|>\(ERROR:)(?:\s*<.+?>\s*)*(.+?)(?:["\']|<|\))' OVR_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' LINK_PATTERN = None #: final download url pattern @@ -126,11 +126,8 @@ class XFSHoster(SimpleHoster): self.req.http.c.setopt(FOLLOWLOCATION, 1) m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) - if m: - if 'down_direct' in data and data['down_direct']: - break - else: - self.error(_("Direct download link detected but not found")) + if m and not "op=" in m.group(1): + break m = re.search(self.LINK_PATTERN, self.html, re.S) if m: @@ -212,7 +209,7 @@ class XFSHoster(SimpleHoster): if m is None: self.errmsg = None else: - self.errmsg = m.group(1) + self.errmsg = m.group(1).strip() self.logWarning(re.sub(r"<.*?>", " ", self.errmsg)) -- cgit v1.2.3 From 2e9b625a215835841bc283cf1f7275a88ea8161b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 18:52:42 +0100 Subject: [SimpleHoster] Fix statusMap generator for python 2.5 --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index a39531b82..ea298cc39 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -16,7 +16,7 @@ from module.utils import fixup, parseFileSize #@TODO: Adapt and move to PyFile in 0.4.10 -statusMap = {v: k for k, v in _statusMap.iteritems()} +statusMap = dict((v, k) for k, v in _statusMap.iteritems()) #@TODO: Remove in 0.4.10 and redirect to self.error instead @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.59" + __version__ = "0.60" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 8e605a8b234974a89c93d38bccae9d19f86e965d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 20:20:49 +0100 Subject: [XFSAccount] Fix trafficleft --- module/plugins/internal/XFSAccount.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 627dc75c4..4a3d0696b 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.26" + __version__ = "0.27" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -71,6 +71,7 @@ class XFSAccount(Account): else: if validuntil > mktime(gmtime()): premium = True + trafficleft = -1 else: premium = False validuntil = None #: registered account type (not premium) @@ -97,9 +98,6 @@ class XFSAccount(Account): except Exception, e: self.logError(e) - else: - if premium: - trafficleft = -1 return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} -- cgit v1.2.3 From 9804b57e02d663328af048a96e60a330ac27b60d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 01:30:45 +0100 Subject: Fixes about create_getInfo --- module/plugins/internal/DeadCrypter.py | 13 ++++++++++++- module/plugins/internal/DeadHoster.py | 21 ++++++++++++--------- module/plugins/internal/SimpleCrypter.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 5 +++-- module/plugins/internal/XFSHoster.py | 4 ++-- 5 files changed, 31 insertions(+), 16 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 4c635e366..c721c8390 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -1,12 +1,15 @@ # -*- coding: utf-8 -*- +from urlparse import urlparse + +from module.plugins.internal.SimpleCrypter import create_getInfo from module.plugins.Crypter import Crypter as _Crypter class DeadCrypter(_Crypter): __name__ = "DeadCrypter" __type__ = "crypter" - __version__ = "0.02" + __version__ = "0.03" __pattern__ = r'^unmatchable$' @@ -15,6 +18,14 @@ class DeadCrypter(_Crypter): __authors__ = [("stickell", "l.stickell@yahoo.it")] + @classmethod + def getInfo(cls, url="", html=""): + return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 1, 'url': url or ""} + + def setup(self): self.pyfile.error = "Crypter is no longer available" self.offline() #@TODO: self.offline("Crypter is no longer available") + + +getInfo = create_getInfo(DeadCrypter) diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index b78068069..b85aea3f9 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -1,20 +1,15 @@ # -*- coding: utf-8 -*- -from module.plugins.Hoster import Hoster as _Hoster - - -def create_getInfo(plugin): +from urlparse import urlparse - def getInfo(urls): - yield map(lambda url: ('#N/A: ' + url, 0, 1, url), urls) - - return getInfo +from module.plugins.internal.SimpleHoster import create_getInfo +from module.plugins.Hoster import Hoster as _Hoster class DeadHoster(_Hoster): __name__ = "DeadHoster" __type__ = "hoster" - __version__ = "0.12" + __version__ = "0.13" __pattern__ = r'^unmatchable$' @@ -23,6 +18,14 @@ class DeadHoster(_Hoster): __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + @classmethod + def getInfo(cls, url="", html=""): + return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 1, 'url': url or ""} + + def setup(self): self.pyfile.error = "Hoster is no longer available" self.offline() #@TODO: self.offline("Hoster is no longer available") + + +getInfo = create_getInfo(DeadHoster) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 5d00daa3c..53ffaf4a6 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -5,14 +5,14 @@ import re from urlparse import urlparse from module.plugins.Crypter import Crypter -from module.plugins.internal.SimpleHoster import SimpleHoster, replace_patterns, set_cookies +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies from module.utils import fixup class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.31" + __version__ = "0.32" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ea298cc39..01702d423 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -100,6 +100,7 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 +#@NOTE: Every plugin must have own parseInfo classmethod to work with 0.4.10 def create_getInfo(plugin): return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfo(urls)] @@ -126,7 +127,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.60" + __version__ = "0.61" __pattern__ = r'^unmatchable$' @@ -197,7 +198,7 @@ class SimpleHoster(Hoster): @classmethod def getInfo(cls, url="", html=""): - info = {'name': url or _("Unknown"), 'size': 0, 'status': 3, 'url': url} + info = {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url or ""} if not html: if url: diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index d5bbf5c63..2cc4833e6 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.22" + __version__ = "0.23" __pattern__ = r'^unmatchable$' @@ -108,7 +108,7 @@ class XFSHoster(SimpleHoster): def getDownloadLink(self): - for i in xrange(1, 5): + for i in xrange(1, 6): self.logDebug("Getting download link: #%d" % i) self.checkErrors() -- cgit v1.2.3 From b5a88f7283fbf3c6177ca4d87e27accc893ea0a8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 05:56:48 +0100 Subject: Fix URL_REPLACEMENTS in XFSHoster plugins --- module/plugins/internal/XFSHoster.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 2cc4833e6..30864ef02 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.23" + __version__ = "0.24" __pattern__ = r'^unmatchable$' @@ -30,8 +30,6 @@ class XFSHoster(SimpleHoster): HOSTER_DOMAIN = None HOSTER_NAME = None - URL_REPLACEMENTS = [(r'/(?:embed-)?(\w{12}).*', r'/\1')] #: plus support embedded files - TEXT_ENCODING = False COOKIES = [(HOSTER_DOMAIN, "lang", "english")] CHECK_DIRECT_LINK = None -- cgit v1.2.3 From 77d012daf519366eb28f1952c8665f7b893f8692 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 15:49:43 +0100 Subject: [XFSAccount] Fix validuntil computation --- module/plugins/internal/XFSAccount.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 4a3d0696b..75111025b 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -2,7 +2,7 @@ import re -from time import gmtime, mktime, strptime +from time import localtime, mktime, strptime from urlparse import urljoin from module.plugins.Account import Account @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.27" + __version__ = "0.28" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -63,18 +63,22 @@ class XFSAccount(Account): self.logDebug("Expire date: " + expiredate) try: - validuntil = mktime(strptime(expiredate, "%d %B %Y")) + validuntil = mktime(strptime(expiredate, "%d %B %Y")) + 86400 except Exception, e: self.logError(e) else: - if validuntil > mktime(gmtime()): + self.logDebug("Valid until: %s" % validuntil) + + if validuntil > mktime(localtime()): premium = True trafficleft = -1 else: premium = False validuntil = None #: registered account type (not premium) + else: + self.logDebug("VALID_UNTIL_PATTERN not found") m = re.search(self.TRAFFIC_LEFT_PATTERN, html) if m: @@ -98,6 +102,8 @@ class XFSAccount(Account): except Exception, e: self.logError(e) + else: + self.logDebug("TRAFFIC_LEFT_PATTERN not found") return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} -- cgit v1.2.3 From 01c55c0168f8745f5e113c88e6c95719a46de6d4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 17:13:34 +0100 Subject: [XFSAccount] Leech traffic support --- module/plugins/internal/XFSAccount.py | 43 ++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 75111025b..54ca5a8d8 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.28" + __version__ = "0.29" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -32,6 +32,9 @@ class XFSAccount(Account): TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found + LEECH_TRAFFIC_PATTERN = r'Leech Traffic left:.*?(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' + LEECH_TRAFFIC_UNIT = "MB" #: used only if no group was found + LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' @@ -49,9 +52,10 @@ class XFSAccount(Account): def loadAccountInfo(self, user, req): - validuntil = None - trafficleft = None - premium = None + validuntil = None + trafficleft = None + leechtraffic = None + premium = None html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) @@ -84,7 +88,7 @@ class XFSAccount(Account): if m: try: traffic = m.groupdict() - size = traffic['S'] + size = traffic['S'] if "nlimited" in size: trafficleft = -1 @@ -105,7 +109,34 @@ class XFSAccount(Account): else: self.logDebug("TRAFFIC_LEFT_PATTERN not found") - return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} + m = re.finditer(self.LEECH_TRAFFIC_PATTERN, html) + if m: + leechtraffic = 0 + try: + for leech in m: + size = leech['S'] + + if "nlimited" in size: + leechtraffic = -1 + if validuntil is None: + validuntil = -1 + break + else: + if 'U' in leech: + unit = leech['U'] + elif isinstance(self.LEECH_TRAFFIC_UNIT, basestring): + unit = self.LEECH_TRAFFIC_UNIT + else: + unit = "" + + leechtraffic += self.parseTraffic(size + unit) + + except Exception, e: + self.logError(e) + else: + self.logDebug("LEECH_TRAFFIC_PATTERN not found") + + return {'validuntil': validuntil, 'trafficleft': trafficleft, 'leechtraffic': leechtraffic, 'premium': premium} def login(self, user, data, req): -- cgit v1.2.3 From ecfeeb64458e2425420d145bae1da89bd6da9280 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 17:29:06 +0100 Subject: [XFSAccount] Revert validuntil computation --- module/plugins/internal/XFSAccount.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 54ca5a8d8..f2a4e8a15 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -2,7 +2,7 @@ import re -from time import localtime, mktime, strptime +from time import gmtime, mktime, strptime from urlparse import urljoin from module.plugins.Account import Account @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.29" + __version__ = "0.30" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -67,7 +67,7 @@ class XFSAccount(Account): self.logDebug("Expire date: " + expiredate) try: - validuntil = mktime(strptime(expiredate, "%d %B %Y")) + 86400 + validuntil = mktime(strptime(expiredate, "%d %B %Y")) except Exception, e: self.logError(e) @@ -75,7 +75,7 @@ class XFSAccount(Account): else: self.logDebug("Valid until: %s" % validuntil) - if validuntil > mktime(localtime()): + if validuntil > mktime(gmtime()): premium = True trafficleft = -1 else: -- cgit v1.2.3 From 2b14c07bfe427f40288b2c73db1813c0dc11659a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 23:18:39 +0100 Subject: [CaptchaService] More debug logging --- module/plugins/internal/CaptchaService.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 38775f519..7009e6986 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -7,7 +7,7 @@ from random import random class CaptchaService: __name__ = "CaptchaService" - __version__ = "0.14" + __version__ = "0.15" __description__ = """Base captcha service plugin""" __license__ = "GPLv3" @@ -52,7 +52,7 @@ class CaptchaService: class ReCaptcha(CaptchaService): __name__ = "ReCaptcha" - __version__ = "0.07" + __version__ = "0.08" __description__ = """ReCaptcha captcha service plugin""" __license__ = "GPLv3" @@ -92,7 +92,6 @@ class ReCaptcha(CaptchaService): raise TypeError(errmsg) js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}) - try: challenge = re.search("challenge : '(.+?)',", js).group(1) server = re.search("server : '(.+?)',", js).group(1) @@ -101,6 +100,8 @@ class ReCaptcha(CaptchaService): result = self.result(server, challenge) + self.plugin.logDebug("ReCaptcha result: %s" % result, "challenge: %s" % challenge) + return challenge, result @@ -111,7 +112,7 @@ class ReCaptcha(CaptchaService): class AdsCaptcha(CaptchaService): __name__ = "AdsCaptcha" - __version__ = "0.04" + __version__ = "0.05" __description__ = """AdsCaptcha captcha service plugin""" __license__ = "GPLv3" @@ -154,7 +155,6 @@ class AdsCaptcha(CaptchaService): CaptchaId, PublicKey = key js = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}) - try: challenge = re.search("challenge: '(.+?)',", js).group(1) server = re.search("server: '(.+?)',", js).group(1) @@ -163,6 +163,8 @@ class AdsCaptcha(CaptchaService): result = self.result(server, challenge) + self.plugin.logDebug("AdsCaptcha result: %s" % result, "challenge: %s" % challenge) + return challenge, result @@ -173,7 +175,7 @@ class AdsCaptcha(CaptchaService): class SolveMedia(CaptchaService): __name__ = "SolveMedia" - __version__ = "0.05" + __version__ = "0.06" __description__ = """SolveMedia captcha service plugin""" __license__ = "GPLv3" @@ -202,6 +204,8 @@ class SolveMedia(CaptchaService): result = self.result(server, challenge) + self.plugin.logDebug("SolveMedia result: %s" % result, "challenge: %s" % challenge) + return challenge, result -- cgit v1.2.3 From 65d290c21e5426f515684f624b8872dec7c0bec0 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 23 Nov 2014 23:20:53 +0100 Subject: [SimpleHoster] Fix _getDirectLink routine --- module/plugins/internal/SimpleHoster.py | 37 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 19 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 01702d423..80ec2669b 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -19,7 +19,6 @@ from module.utils import fixup, parseFileSize statusMap = dict((v, k) for k, v in _statusMap.iteritems()) -#@TODO: Remove in 0.4.10 and redirect to self.error instead def _error(self, reason, type): if not reason and not type: type = "unknown" @@ -57,12 +56,13 @@ def parseHtmlTagAttrValue(attr_name, tag): return m.group(2) if m else None -def parseHtmlForm(attr_str, html, input_names=None): - for form in re.finditer(r"(?P]*%s[^>]*>)(?P.*?)]*>" % attr_str, +def parseHtmlForm(attr_str, html, input_names={}): + for form in re.finditer(r"(?P]*%s[^>]*>)(?P.*?)]*>" % attr_str, html, re.S | re.I): inputs = {} - action = parseHtmlTagAttrValue("action", form.group('tag')) - for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=]*>)([^<]*(?= Date: Mon, 24 Nov 2014 01:12:54 +0100 Subject: [XFSHoster] Code improvements --- module/plugins/internal/SimpleHoster.py | 3 ++- module/plugins/internal/XFSHoster.py | 35 +++++++++++++++++++-------------- 2 files changed, 22 insertions(+), 16 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 80ec2669b..95b52a017 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -19,6 +19,7 @@ from module.utils import fixup, parseFileSize statusMap = dict((v, k) for k, v in _statusMap.iteritems()) +#@TODO: Remove in 0.4.10 and redirect to self.error instead def _error(self, reason, type): if not reason and not type: type = "unknown" @@ -479,7 +480,7 @@ class SimpleHoster(Hoster): self.retry(max_tries=max_tries, reason=_("Download limit reached")) - def parseHtmlForm(self, attr_str='', input_names={}): + def parseHtmlForm(self, attr_str="", input_names={}): return parseHtmlForm(attr_str, self.html, input_names) diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 30864ef02..2aaf18b1a 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.24" + __version__ = "0.26" __pattern__ = r'^unmatchable$' @@ -33,10 +33,10 @@ class XFSHoster(SimpleHoster): TEXT_ENCODING = False COOKIES = [(HOSTER_DOMAIN, "lang", "english")] CHECK_DIRECT_LINK = None - MULTI_HOSTER = False + MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' - NAME_PATTERN = r'(>Filename:|name="fname" value="||<[Tt]itle>.*?Download )(?P.+?)(\s*<|")' + NAME_PATTERN = r'(>Filename:|name="fname" value="|)(?P.+?)(\s*<|")' SIZE_PATTERN = r'(>Size:|>File:.*>|)(?P[\d.,]+)\s*(?P[\w^_]+)' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' @@ -46,14 +46,17 @@ class XFSHoster(SimpleHoster): PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only' ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter>|>Error|>\(ERROR:)(?:\s*<.+?>\s*)*(.+?)(?:["\']|<|\))' - OVR_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' - LINK_PATTERN = None #: final download url pattern + LEECH_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' + LINK_PATTERN = None #: final download url pattern CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' CAPTCHA_DIV_PATTERN = r'>Enter code.*?(.+?)
' RECAPTCHA_PATTERN = None SOLVEMEDIA_PATTERN = None + FORM_PATTERN = None + FORM_INPUTS_MAP = None #: dict passed as input_names to parseHtmlForm + def setup(self): self.chunkLimit = 1 @@ -143,7 +146,7 @@ class XFSHoster(SimpleHoster): #only tested with easybytez.com self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) - action, inputs = self.parseHtmlForm('') + action, inputs = self.parseHtmlForm() upload_id = "%012d" % int(random() * 10 ** 12) action += upload_id + "&js_on=1&utype=prem&upload_type=url" @@ -184,11 +187,11 @@ class XFSHoster(SimpleHoster): self.fail(stmsg) #get easybytez.com link for uploaded file - m = re.search(self.OVR_LINK_PATTERN, self.html) + m = re.search(self.LEECH_LINK_PATTERN, self.html) if m is None: - self.error(_("OVR_LINK_PATTERN not found")) + self.error(_("LEECH_LINK_PATTERN not found")) - header = self.load(m.group(1).strip(), just_header=True, decode=True) #@TODO: Remove .strip() in 0.4.10 + header = self.load(m.group(1), just_header=True, decode=True) if 'location' in header: #: Direct download link self.link = header['location'] @@ -257,10 +260,10 @@ class XFSHoster(SimpleHoster): def getPostParameters(self): - if hasattr(self, "FORM_PATTERN"): - action, inputs = self.parseHtmlForm(self.FORM_PATTERN) + if self.FORM_PATTERN or self.FORM_INPUTS_MAP: + action, inputs = self.parseHtmlForm(self.FORM_PATTERN or "", self.FORM_INPUTS_MAP or {}) else: - action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) + action, inputs = self.parseHtmlForm(input_names={'op': re.compile(r'^download')}) if not inputs: action, inputs = self.parseHtmlForm('F1') @@ -311,8 +314,8 @@ class XFSHoster(SimpleHoster): m = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.S) if m: captcha_div = m.group(1) + numerals = re.findall(r'(\d)', html_unescape(captcha_div)) self.logDebug(captcha_div) - numerals = re.findall(r'(\d)', html_unescape(captcha_div)) inputs['code'] = "".join([a[1] for a in sorted(numerals, key=lambda num: int(num[0]))]) self.logDebug("Captcha code: %s" % inputs['code'], numerals) return 2 @@ -322,9 +325,10 @@ class XFSHoster(SimpleHoster): captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) except: captcha_key = recaptcha.detect_key() + else: + self.logDebug("ReCaptcha key: %s" % captcha_key) if captcha_key: - self.logDebug("ReCaptcha key: %s" % captcha_key) inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) return 3 @@ -333,9 +337,10 @@ class XFSHoster(SimpleHoster): captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) except: captcha_key = solvemedia.detect_key() + else: + self.logDebug("SolveMedia key: %s" % captcha_key) if captcha_key: - self.logDebug("SolveMedia key: %s" % captcha_key) inputs['adcopy_challenge'], inputs['adcopy_response'] = solvemedia.challenge(captcha_key) return 4 -- cgit v1.2.3 From 0d27fda4b6edbfcba9b78b2c3f91063b8464455a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 28 Nov 2014 20:40:00 +0100 Subject: [XFSAccount] Fix leech traffic detection --- module/plugins/internal/XFSAccount.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index f2a4e8a15..5a265c08a 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.30" + __version__ = "0.31" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -109,12 +109,12 @@ class XFSAccount(Account): else: self.logDebug("TRAFFIC_LEFT_PATTERN not found") - m = re.finditer(self.LEECH_TRAFFIC_PATTERN, html) - if m: + leech = [m.groupdict() for m in re.finditer(self.LEECH_TRAFFIC_PATTERN, html)] + if leech: leechtraffic = 0 try: - for leech in m: - size = leech['S'] + for traffic in leech: + size = traffic['S'] if "nlimited" in size: leechtraffic = -1 @@ -122,8 +122,8 @@ class XFSAccount(Account): validuntil = -1 break else: - if 'U' in leech: - unit = leech['U'] + if 'U' in traffic: + unit = traffic['U'] elif isinstance(self.LEECH_TRAFFIC_UNIT, basestring): unit = self.LEECH_TRAFFIC_UNIT else: -- cgit v1.2.3 From 8319b2fa98ee0408b4886ff474604e8199901f5c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 28 Nov 2014 21:16:49 +0100 Subject: [SimpleHoster] Fix deprecated method getFileInfo --- module/plugins/internal/SimpleHoster.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 95b52a017..d74da1301 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.62" + __version__ = "0.63" __pattern__ = r'^unmatchable$' @@ -315,8 +315,7 @@ class SimpleHoster(Hoster): premium_only = 'error' in self.info and self.info['error'] == "premium-only" - info = self.getInfo(pyfile.url, self.html) - self._updateInfo(info) + self._updateInfo(self.getInfo(pyfile.url, self.html)) self.checkNameSize() @@ -408,7 +407,9 @@ class SimpleHoster(Hoster): #: Deprecated def getFileInfo(self): - return self.checkInfo() + self.info = {} + self.checkInfo() + return self.info def _updateInfo(self, info): -- cgit v1.2.3 From a85526b5a3b1683b928ecfe9d93ec953dc29dae7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 28 Nov 2014 22:29:40 +0100 Subject: [SimpleHoster] Improve getInfo routine --- module/plugins/internal/SimpleHoster.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index d74da1301..529abce9a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.63" + __version__ = "0.64" __pattern__ = r'^unmatchable$' @@ -224,13 +224,19 @@ class SimpleHoster(Hoster): except: pass - for pattern in ("INFO_PATTERN", "NAME_PATTERN", "SIZE_PATTERN", - "FILE_INFO_PATTERN", "FILE_NAME_PATTERN", "FILE_SIZE_PATTERN"): #@TODO: Remove in 0.4.10 + for pattern in ("FILE_INFO_PATTERN", "INFO_PATTERN", + "FILE_NAME_PATTERN", "NAME_PATTERN", + "FILE_SIZE_PATTERN", "SIZE_PATTERN"): #@TODO: Remove old patterns starting with "FILE_" in 0.4.10 try: attr = getattr(cls, pattern) - info.update(re.search(attr, html).groupdict()) + dict = re.search(attr, html).groupdict() + + if all(True for k in dict if k not in info): + info.update(dict) + except AttributeError: continue + else: online = True -- cgit v1.2.3 From f3c71c3c2a6ace91cd7d72217bc218f1ae976a6a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 28 Nov 2014 23:36:37 +0100 Subject: [UnRar] Fix version detection --- module/plugins/internal/UnRar.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 12bd84d20..c15a4c96e 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(AbtractExtractor): __name__ = "UnRar" - __version__ = "0.18" + __version__ = "0.19" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -32,12 +32,12 @@ class UnRar(AbtractExtractor): CMD = "unrar" # there are some more uncovered rar formats - re_version = re.compile(r"(UNRAR 5[\d.]+(.*?)freeware)") - re_splitfile = re.compile(r"(.*)\.part(\d+)\.rar$", re.I) - re_partfiles = re.compile(r".*\.(rar|r\d+)", re.I) - re_filelist = re.compile(r"(.+)\s+(\d+)\s+(\d+)\s+") - re_filelist5 = re.compile(r"(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)") - re_wrongpwd = re.compile("(Corrupt file or wrong password|password incorrect)", re.I) + re_version = re.compile(r'UNRAR ([\w .]+?) freeware') + re_splitfile = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) + re_partfiles = re.compile(r'.*\.(rar|r\d+)', re.I) + re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+') + re_filelist5 = re.compile(r'(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') + re_wrongpwd = re.compile(r'(Corrupt file or wrong password|password incorrect)', re.I) @staticmethod -- cgit v1.2.3 From 42bb9542bb2dfeafcf98630cf4a13a661a060398 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 29 Nov 2014 00:29:15 +0100 Subject: [SimpleHoster] Fix _getDirectLink --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 529abce9a..bdd84b74b 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -117,7 +117,7 @@ def _getDirectLink(self, url): if not 'location' in header or not header['location']: return "" - if header['code'] != 302 or 'content-type' not in header or header['content-type'] != "text/plain": + if header['code'] != 302 or 'content-type' in header and header['content-type'] != "text/plain": return "" return header['location'] @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.64" + __version__ = "0.65" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 00277a73a3754a58d1e09b44348a75e0dfaa5f61 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 29 Nov 2014 00:47:04 +0100 Subject: [SimpleHoster] Fix _getDirectLink (2) --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index bdd84b74b..274d95154 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -117,7 +117,7 @@ def _getDirectLink(self, url): if not 'location' in header or not header['location']: return "" - if header['code'] != 302 or 'content-type' in header and header['content-type'] != "text/plain": + if header['code'] != 302 or 'content-type' in header and "text/plain" not in header['content-type']: return "" return header['location'] @@ -126,7 +126,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.65" + __version__ = "0.66" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From d4719096cad1127540c32bb22345ebcba8617fa9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 29 Nov 2014 23:02:24 +0100 Subject: [SimpleHoster] Fix _getDirectLink (3) --- module/plugins/internal/SimpleHoster.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 274d95154..24a2fa6b0 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -5,8 +5,6 @@ import re from time import time from urlparse import urlparse -from pycurl import FOLLOWLOCATION - from module.PyFile import statusMap as _statusMap from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL @@ -34,7 +32,11 @@ def _error(self, reason, type): #@TODO: Remove in 0.4.10 def _wait(self, seconds, reconnect): if seconds: - self.setWait(seconds, reconnect) + self.setWait(seconds) + + if reconnect is not None: + self.wantReconnect = reconnect + super(SimpleHoster, self).wait() @@ -114,19 +116,22 @@ def timestamp(): def _getDirectLink(self, url): header = self.load(url, ref=True, just_header=True, decode=True) - if not 'location' in header or not header['location']: + if not 'code' in header or header['code'] != 302: return "" - if header['code'] != 302 or 'content-type' in header and "text/plain" not in header['content-type']: + if not 'location' in header or not header['location']: return "" + # if 'content-type' in header and "text/plain" not in header['content-type']: + # return "" + return header['location'] class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.66" + __version__ = "0.67" __pattern__ = r'^unmatchable$' @@ -197,7 +202,7 @@ class SimpleHoster(Hoster): @classmethod def getInfo(cls, url="", html=""): - info = {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url or ""} + info = {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3 if url else 1, 'url': url or ""} if not html: if url: -- cgit v1.2.3 From bc3f5f4903444e438a8a07688c82f0c2995dafa9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 30 Nov 2014 21:15:54 +0100 Subject: [SimpleHoster] Unquote file name --- module/plugins/internal/SimpleHoster.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 24a2fa6b0..f741cd691 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -3,6 +3,7 @@ import re from time import time +from urllib import unquote from urlparse import urlparse from module.PyFile import statusMap as _statusMap @@ -131,7 +132,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.67" + __version__ = "0.68" __pattern__ = r'^unmatchable$' @@ -249,7 +250,7 @@ class SimpleHoster(Hoster): info['status'] = 2 if 'N' in info: - info['name'] = replace_patterns(info['N'].strip(), + info['name'] = replace_patterns(unquote(info['N'].strip()), cls.FILE_NAME_REPLACEMENTS if hasattr(cls, "FILE_NAME_REPLACEMENTS") else cls.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 if 'S' in info: -- cgit v1.2.3 From 45c7759a45e1e9771eec6fc231b523aaa74821a4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 1 Dec 2014 18:07:19 +0100 Subject: [DeadCrypter][DeadHoster] Improve getInfo --- module/plugins/internal/DeadCrypter.py | 5 +++-- module/plugins/internal/DeadHoster.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index c721c8390..07c5c3881 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from urllib import unquote from urlparse import urlparse from module.plugins.internal.SimpleCrypter import create_getInfo @@ -9,7 +10,7 @@ from module.plugins.Crypter import Crypter as _Crypter class DeadCrypter(_Crypter): __name__ = "DeadCrypter" __type__ = "crypter" - __version__ = "0.03" + __version__ = "0.04" __pattern__ = r'^unmatchable$' @@ -20,7 +21,7 @@ class DeadCrypter(_Crypter): @classmethod def getInfo(cls, url="", html=""): - return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 1, 'url': url or ""} + return {'name': urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 1, 'url': url} def setup(self): diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index b85aea3f9..6f3252f70 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from urllib import unquote from urlparse import urlparse from module.plugins.internal.SimpleHoster import create_getInfo @@ -9,7 +10,7 @@ from module.plugins.Hoster import Hoster as _Hoster class DeadHoster(_Hoster): __name__ = "DeadHoster" __type__ = "hoster" - __version__ = "0.13" + __version__ = "0.14" __pattern__ = r'^unmatchable$' @@ -20,7 +21,7 @@ class DeadHoster(_Hoster): @classmethod def getInfo(cls, url="", html=""): - return {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 1, 'url': url or ""} + return {'name': urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 1, 'url': url} def setup(self): -- cgit v1.2.3 From 7ef266b8c5681236111fe3e71acbb7eaeefd543f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 1 Dec 2014 18:10:59 +0100 Subject: [SimpleHoster] Fix some stuff, improve others --- module/plugins/internal/SimpleHoster.py | 123 +++++++++++++++++++++----------- 1 file changed, 83 insertions(+), 40 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index f741cd691..736d15f3a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -4,7 +4,7 @@ import re from time import time from urllib import unquote -from urlparse import urlparse +from urlparse import urljoin, urlparse from module.PyFile import statusMap as _statusMap from module.network.CookieJar import CookieJar @@ -33,7 +33,7 @@ def _error(self, reason, type): #@TODO: Remove in 0.4.10 def _wait(self, seconds, reconnect): if seconds: - self.setWait(seconds) + self.setWait(seconds + 1) if reconnect is not None: self.wantReconnect = reconnect @@ -114,25 +114,35 @@ def timestamp(): #@TODO: Move to hoster class in 0.4.10 -def _getDirectLink(self, url): +def _isDirectLink(self, url, resumable=True): header = self.load(url, ref=True, just_header=True, decode=True) - if not 'code' in header or header['code'] != 302: - return "" - if not 'location' in header or not header['location']: return "" - # if 'content-type' in header and "text/plain" not in header['content-type']: - # return "" + location = header['location'] + + if resumable: #: sometimes http code may be wrong... + if self.load(location, ref=True, cookies=True, just_header=True, decode=True)['location']: + return "" + else: + if not 'code' in header or header['code'] != 302: + return "" - return header['location'] + if urlparse(location).scheme: + link = location + else: + p = urlparse(url) + base = "%s://%s" % (p.scheme, p.netloc) + link = urljoin(base, location) + + return link class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.68" + __version__ = "0.69" __pattern__ = r'^unmatchable$' @@ -154,6 +164,9 @@ class SimpleHoster(Hoster): SIZE_PATTERN: (optional) Size that will be checked for the file example: SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' + HASHSUM_PATTERN: (optional) Hash code and type of the file + example: HASHSUM_PATTERN = r'(?Phash_code) (?PMD5)' + OFFLINE_PATTERN: (optional) Check if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' @@ -189,9 +202,9 @@ class SimpleHoster(Hoster): TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - CHECK_DIRECT_LINK = None #: when None self-set to True if self.account else False - MULTI_HOSTER = False #: Set to True to leech other hoster link - CONTENT_DISPOSITION = False #: Set to True to replace file name with content-disposition value in http header + CHECK_DIRECT_LINK = None #: Set to True to check for direct link, set to None to do it only if self.account is True + MULTI_HOSTER = False #: Set to True to leech other hoster link (according its multihoster hook if available) + CONTENT_DISPOSITION = False #: Set to True to replace file name with content-disposition value from http header @classmethod @@ -203,14 +216,32 @@ class SimpleHoster(Hoster): @classmethod def getInfo(cls, url="", html=""): - info = {'name': urlparse(url).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3 if url else 1, 'url': url or ""} + info = {'name': urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url} if not html: - if url: - html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING) - if isinstance(cls.TEXT_ENCODING, basestring): - html = unicode(html, cls.TEXT_ENCODING) - else: + try: + if not url: + info['error'] = "missing url" + info['status'] = 1 + raise + + try: + html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING) + + if isinstance(cls.TEXT_ENCODING, basestring): + html = unicode(html, cls.TEXT_ENCODING) + + except BadHeader, e: + info['error'] = "%d: %s" % (e.code, e.content) + + if e.code is 404: + info['status'] = 1 + raise + + if e.code is 503: + info['status'] = 6 + raise + except: return info online = False @@ -226,19 +257,20 @@ class SimpleHoster(Hoster): else: try: - info.update(re.match(cls.__pattern__, url).groupdict()) + info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here, please save api stuff to info['api'] except: pass for pattern in ("FILE_INFO_PATTERN", "INFO_PATTERN", "FILE_NAME_PATTERN", "NAME_PATTERN", - "FILE_SIZE_PATTERN", "SIZE_PATTERN"): #@TODO: Remove old patterns starting with "FILE_" in 0.4.10 + "FILE_SIZE_PATTERN", "SIZE_PATTERN", + "HASHSUM_PATTERN"): #@TODO: Remove old patterns starting with "FILE_" in 0.4.10 try: attr = getattr(cls, pattern) dict = re.search(attr, html).groupdict() - if all(True for k in dict if k not in info): - info.update(dict) + if all(True for k in dict if k not in info['pattern']): + info['pattern'].update(dict) except AttributeError: continue @@ -249,12 +281,12 @@ class SimpleHoster(Hoster): if online: info['status'] = 2 - if 'N' in info: - info['name'] = replace_patterns(unquote(info['N'].strip()), + if 'N' in info['pattern']: + info['name'] = replace_patterns(unquote(info['pattern']['N'].strip()), cls.FILE_NAME_REPLACEMENTS if hasattr(cls, "FILE_NAME_REPLACEMENTS") else cls.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 - if 'S' in info: - size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], + if 'S' in info['pattern']: + size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info else info['pattern']['S'], cls.FILE_SIZE_REPLACEMENTS if hasattr(cls, "FILE_SIZE_REPLACEMENTS") else cls.SIZE_REPLACEMENTS) #@TODO: Remove FILE_SIZE_REPLACEMENTS check in 0.4.10 info['size'] = parseFileSize(size) @@ -262,6 +294,10 @@ class SimpleHoster(Hoster): unit = info['units'] if 'units' in info else None info['size'] = parseFileSize(info['size'], unit) + if 'H' in info['pattern']: + hashtype = info['pattern']['T'] if 'T' in info['pattern'] else "hash" + info[hashtype] = info['pattern']['H'] + return info @@ -349,15 +385,20 @@ class SimpleHoster(Hoster): if self.link: self.download(self.link, disposition=self.CONTENT_DISPOSITION) + self.checkFile() + + + def checkFile(self) + if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": #@TODO: Move to hoster in 0.4.10 + self.fail(_("Empty file")) + def checkErrors(self): - if hasattr(self, 'WAIT_PATTERN'): - m = re.search(self.WAIT_PATTERN, self.html) + if hasattr(self, 'ERROR_PATTERN'): + m = re.search(self.ERROR_PATTERN, self.html) if m: - wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in - re.findall(r'(\d+)\s*(hr|hour|min|sec)', m, re.I)]) - self.wait(wait_time, False) - return + e = self.info['error'] = m.group(1) + self.error(e) if hasattr(self, 'PREMIUM_ONLY_PATTERN'): m = re.search(self.PREMIUM_ONLY_PATTERN, self.html) @@ -365,11 +406,13 @@ class SimpleHoster(Hoster): self.info['error'] = "premium-only" return - if hasattr(self, 'ERROR_PATTERN'): - m = re.search(self.ERROR_PATTERN, self.html) + if hasattr(self, 'WAIT_PATTERN'): + m = re.search(self.WAIT_PATTERN, self.html) if m: - e = self.info['error'] = m.group(1) - self.error(e) + wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec)', m, re.I)]) + self.wait(wait_time, False) + return self.info.pop('error', None) @@ -431,7 +474,7 @@ class SimpleHoster(Hoster): def handleDirect(self): - link = _getDirectLink(self, self.pyfile.url) + link = _isDirectLink(self, self.pyfile.url, self.resumeDownload) if link: self.logInfo(_("Direct download link detected")) @@ -460,7 +503,7 @@ class SimpleHoster(Hoster): self.link = m.group(1) except Exception, e: - self.fail(str(e)) + self.fail(e) def handlePremium(self): @@ -475,7 +518,7 @@ class SimpleHoster(Hoster): self.link = m.group(1) except Exception, e: - self.fail(str(e)) + self.fail(e) def longWait(self, wait_time=None, max_tries=3): -- cgit v1.2.3 From 86d3b6249073947132ed3a9eeb1b1e987d19569a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 1 Dec 2014 18:17:13 +0100 Subject: Update some plugins --- module/plugins/internal/XFSHoster.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 2aaf18b1a..587d47f6e 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.26" + __version__ = "0.27" __pattern__ = r'^unmatchable$' @@ -49,10 +49,10 @@ class XFSHoster(SimpleHoster): LEECH_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' LINK_PATTERN = None #: final download url pattern - CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' - CAPTCHA_DIV_PATTERN = r'>Enter code.*?(.+?)
' - RECAPTCHA_PATTERN = None - SOLVEMEDIA_PATTERN = None + CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' + CAPTCHA_BLOCK_PATTERN = r'>Enter code.*?(.+?)
Date: Thu, 25 Sep 2014 20:59:30 +0200 Subject: Extend ExtractArchive events + code optimizations https://github.com/pyload/pyload/pull/622 https://github.com/pyload/pyload/pull/726 --- module/plugins/internal/UnRar.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 19c278735..91be69433 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -4,7 +4,7 @@ import os import re from glob import glob -from os.path import join +from os import basename, join from string import digits from subprocess import Popen, PIPE @@ -12,14 +12,23 @@ from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPas from module.utils import save_join, decode +def renice(pid, value): + if os.name != "nt" and value: + try: + Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1) + except: + print "Renice failed" + + class UnRar(AbtractExtractor): __name__ = "UnRar" - __version__ = "0.16" + __version__ = "0.17" __description__ = """Rar extractor plugin""" __author_name__ = "RaNaN" __author_mail__ = "RaNaN@pyload.org" + CMD = "unrar" # there are some more uncovered rar formats @@ -50,6 +59,7 @@ class UnRar(AbtractExtractor): return True + @staticmethod def getTargets(files_ids): result = [] @@ -68,12 +78,14 @@ class UnRar(AbtractExtractor): return result + def init(self): self.passwordProtected = False self.headerProtected = False #: list files will not work without password self.smallestFile = None #: small file to test passwords self.password = "" #: save the correct password + def checkArchive(self): p = self.call_unrar("l", "-v", self.file) out, err = p.communicate() @@ -100,6 +112,7 @@ class UnRar(AbtractExtractor): return False + def checkPassword(self, password): # at this point we can only verify header protected files if self.headerProtected: @@ -110,6 +123,7 @@ class UnRar(AbtractExtractor): return True + def extract(self, progress, password=None): command = "x" if self.fullpath else "e" @@ -151,13 +165,15 @@ class UnRar(AbtractExtractor): self.password = password self.listContent() + def getDeleteFiles(self): - if ".part" in self.file: + if ".part" in basename(self.file): return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.IGNORECASE)) # get files which matches .r* and filter unsuited files out parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.IGNORECASE)) return filter(lambda x: self.re_partfiles.match(x), parts) + def listContent(self): command = "vb" if self.fullpath else "lb" p = self.call_unrar(command, "-v", self.file, password=self.password) @@ -177,6 +193,7 @@ class UnRar(AbtractExtractor): self.files = result + def call_unrar(self, command, *xargs, **kwargs): args = [] # overwrite flag @@ -202,11 +219,3 @@ class UnRar(AbtractExtractor): p = Popen(call, stdout=PIPE, stderr=PIPE) return p - - -def renice(pid, value): - if os.name != "nt" and value: - try: - Popen(["renice", str(value), str(pid)], stdout=PIPE, stderr=PIPE, bufsize=-1) - except: - print "Renice failed" -- cgit v1.2.3 From 76c67b22c0b2dbd275fae77e8a3121acb2d4301b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Sep 2014 15:39:33 +0200 Subject: Fix cookie deletion + Account support for SimpleCrypter Fix: https://github.com/pyload/pyload/issues/749 --- module/plugins/internal/SimpleCrypter.py | 17 ++++++++++++++--- module/plugins/internal/SimpleHoster.py | 23 +++++++++++++++++------ 2 files changed, 31 insertions(+), 9 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 3cd52d9f1..bacfce1dd 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.utils import html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.10" + __version__ = "0.11" __pattern__ = None @@ -48,9 +48,14 @@ class SimpleCrypter(Crypter): return the html of the page number page_n """ + URL_REPLACEMENTS = [] - SH_COOKIES = True # or False or list of tuples [(domain, name, value)] + SH_BROKEN_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct + SH_COOKIES = True #: or False or list of tuples [(domain, name, value)] + + LOGIN_ACCOUNT = False + LOGIN_PREMIUM = False def setup(self): @@ -59,9 +64,15 @@ class SimpleCrypter(Crypter): def decrypt(self, pyfile): + if self.LOGIN_ACCOUNT and not self.account: + self.fail('Required account not found!') + + if self.LOGIN_PREMIUM and not self.premium: + self.fail('Required premium account not found!') + pyfile.url = replace_patterns(pyfile.url, self.URL_REPLACEMENTS) - self.html = self.load(pyfile.url, decode=True) + self.html = self.load(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) self.checkOnline() diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ec9cf1b70..b2db56f3f 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -187,24 +187,26 @@ class SimpleHoster(Hoster): FILE_SIZE_REPLACEMENTS = [] FILE_URL_REPLACEMENTS = [] - SH_BROKEN_ENCODING = False # Set to True or encoding name if encoding in http header is not correct - SH_COOKIES = True # or False or list of tuples [(domain, name, value)] - SH_CHECK_TRAFFIC = False # True = force check traffic left for a premium account + SH_BROKEN_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct + SH_COOKIES = True #: or False or list of tuples [(domain, name, value)] + SH_CHECK_TRAFFIC = False #: True = force check traffic left for a premium account def init(self): self.file_info = {} + def setup(self): self.resumeDownload = self.multiDL = self.premium if isinstance(self.SH_COOKIES, list): set_cookies(self.req.cj, self.SH_COOKIES) + def process(self, pyfile): pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS) self.req.setOption("timeout", 120) # Due to a 0.4.9 core bug self.load would keep previous cookies even if overridden by cookies parameter. - # Workaround using getURL. Can be reverted in 0.5 as the cookies bug has been fixed. + # Workaround using getURL. Can be reverted in 0.4.10 as the cookies bug has been fixed. self.html = getURL(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) if not premium_only: # Usually premium only pages doesn't show the file information @@ -215,16 +217,18 @@ class SimpleHoster(Hoster): elif premium_only: self.fail("This link require a premium account") else: - # This line is required due to the getURL workaround. Can be removed in 0.5 + # This line is required due to the getURL workaround. Can be removed in 0.4.10 self.html = self.load(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) self.handleFree() + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): if type(url) == unicode: url = url.encode('utf8') return Hoster.load(self, url=url, get=get, post=post, ref=ref, cookies=cookies, just_header=just_header, decode=decode) + def getFileInfo(self): self.logDebug("URL: %s" % self.pyfile.url) @@ -251,15 +255,19 @@ class SimpleHoster(Hoster): self.logDebug("FILE NAME: %s FILE SIZE: %s" % (self.pyfile.name, self.pyfile.size)) return self.file_info + def handleFree(self): self.fail("Free download not implemented") + def handlePremium(self): self.fail("Premium download not implemented") + def parseError(self, msg): raise PluginParseError(msg) + def longWait(self, wait_time=None, max_tries=3): if wait_time and isinstance(wait_time, (int, long, float)): time_str = "%dh %dm" % divmod(wait_time / 60, 60) @@ -274,9 +282,11 @@ class SimpleHoster(Hoster): self.wait() self.retry(max_tries=max_tries, reason="Download limit reached") + def parseHtmlForm(self, attr_str='', input_names=None): return parseHtmlForm(attr_str, self.html, input_names) + def checkTrafficLeft(self): traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] if traffic == -1: @@ -285,7 +295,8 @@ class SimpleHoster(Hoster): self.logInfo("Filesize: %i KiB, Traffic left for user %s: %i KiB" % (size, self.user, traffic)) return size <= traffic - # TODO: Remove in 0.5 + + #@TODO: Remove in 0.4.10 def wait(self, seconds=False, reconnect=False): if seconds: self.setWait(seconds, reconnect) -- cgit v1.2.3 From 4709a41c176dd2947709b7f35ccc84213d574624 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Sep 2014 16:59:26 +0200 Subject: Rename SH flags --- module/plugins/internal/SimpleCrypter.py | 10 +++++----- module/plugins/internal/SimpleHoster.py | 28 ++++++++++++++-------------- 2 files changed, 19 insertions(+), 19 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index bacfce1dd..184ba66c9 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -51,16 +51,16 @@ class SimpleCrypter(Crypter): URL_REPLACEMENTS = [] - SH_BROKEN_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct - SH_COOKIES = True #: or False or list of tuples [(domain, name, value)] + TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct + COOKIES = True #: or False or list of tuples [(domain, name, value)] LOGIN_ACCOUNT = False LOGIN_PREMIUM = False def setup(self): - if isinstance(self.SH_COOKIES, list): - set_cookies(self.req.cj, self.SH_COOKIES) + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) def decrypt(self, pyfile): @@ -72,7 +72,7 @@ class SimpleCrypter(Crypter): pyfile.url = replace_patterns(pyfile.url, self.URL_REPLACEMENTS) - self.html = self.load(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) + self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) self.checkOnline() diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index b2db56f3f..6ecb2d9f2 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -78,8 +78,8 @@ def parseFileInfo(self, url='', html=''): else: if not html and hasattr(self, "html"): html = self.html - if isinstance(self.SH_BROKEN_ENCODING, (str, unicode)): - html = unicode(html, self.SH_BROKEN_ENCODING) + if isinstance(self.TEXT_ENCODING, basestring): + html = unicode(html, self.TEXT_ENCODING) if hasattr(self, "html"): self.html = html @@ -112,7 +112,7 @@ def parseFileInfo(self, url='', html=''): size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], self.FILE_SIZE_REPLACEMENTS) info['size'] = parseFileSize(size) - elif isinstance(info['size'], (str, unicode)): + elif isinstance(info['size'], basestring): if 'units' in info: info['size'] += info['units'] info['size'] = parseFileSize(info['size']) @@ -128,10 +128,10 @@ def create_getInfo(plugin): def getInfo(urls): for url in urls: cj = CookieJar(plugin.__name__) - if isinstance(plugin.SH_COOKIES, list): - set_cookies(cj, plugin.SH_COOKIES) + if isinstance(plugin.COOKIES, list): + set_cookies(cj, plugin.COOKIES) file_info = parseFileInfo(plugin, url, getURL(replace_patterns(url, plugin.FILE_URL_REPLACEMENTS), - decode=not plugin.SH_BROKEN_ENCODING, cookies=cj)) + decode=not plugin.TEXT_ENCODING, cookies=cj)) yield file_info return getInfo @@ -187,9 +187,9 @@ class SimpleHoster(Hoster): FILE_SIZE_REPLACEMENTS = [] FILE_URL_REPLACEMENTS = [] - SH_BROKEN_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct - SH_COOKIES = True #: or False or list of tuples [(domain, name, value)] - SH_CHECK_TRAFFIC = False #: True = force check traffic left for a premium account + TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct + COOKIES = True #: or False or list of tuples [(domain, name, value)] + FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account def init(self): @@ -198,8 +198,8 @@ class SimpleHoster(Hoster): def setup(self): self.resumeDownload = self.multiDL = self.premium - if isinstance(self.SH_COOKIES, list): - set_cookies(self.req.cj, self.SH_COOKIES) + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) def process(self, pyfile): @@ -207,18 +207,18 @@ class SimpleHoster(Hoster): self.req.setOption("timeout", 120) # Due to a 0.4.9 core bug self.load would keep previous cookies even if overridden by cookies parameter. # Workaround using getURL. Can be reverted in 0.4.10 as the cookies bug has been fixed. - self.html = getURL(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) + self.html = getURL(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) if not premium_only: # Usually premium only pages doesn't show the file information self.getFileInfo() - if self.premium and (not self.SH_CHECK_TRAFFIC or self.checkTrafficLeft()): + if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): self.handlePremium() elif premium_only: self.fail("This link require a premium account") else: # This line is required due to the getURL workaround. Can be removed in 0.4.10 - self.html = self.load(pyfile.url, decode=not self.SH_BROKEN_ENCODING, cookies=self.SH_COOKIES) + self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) self.handleFree() -- cgit v1.2.3 From 0c2c1f9e7c514a77fdb04329322fab2ed601df01 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Sep 2014 19:09:47 +0200 Subject: [CaptchaService] Fix ReCaptcha key recognition Fix: https://github.com/pyload/pyload/issues/751 --- module/plugins/internal/CaptchaService.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index b247ba654..26482379d 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -7,7 +7,7 @@ from random import random class CaptchaService: __name__ = "CaptchaService" - __version__ = "0.05" + __version__ = "0.06" __description__ = """Captcha service plugin""" __author_name__ = "pyLoad Team" @@ -28,6 +28,7 @@ class ReCaptcha: def __init__(self, plugin): self.plugin = plugin + def detect_key(self, html): m = re.search(self.RECAPTCHA_KEY_PATTERN, html) if m is None: @@ -38,11 +39,13 @@ class ReCaptcha: else: return None + def challenge(self, key=None): - if key is None and self.recaptcha_key: - key = self.recaptcha_key - else: - raise TypeError("ReCaptcha key not found") + if not key: + if self.recaptcha_key: + key = self.recaptcha_key + else: + raise TypeError("ReCaptcha key not found") js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={"k": key}, cookies=True) @@ -55,6 +58,7 @@ class ReCaptcha: return challenge, result + def result(self, server, challenge): return self.plugin.decryptCaptcha("%simage" % server, get={"c": challenge}, cookies=True, forceUser=True, imgtype="jpg") @@ -74,6 +78,7 @@ class AdsCaptcha(CaptchaService): return challenge, result + def result(self, server, challenge): return self.plugin.decryptCaptcha("%sChallenge.aspx" % server, get={"cid": challenge, "dummy": random()}, cookies=True, imgtype="jpg") @@ -92,5 +97,6 @@ class SolveMedia(CaptchaService): return challenge, result + def result(self, challenge): return self.plugin.decryptCaptcha("http://api.solvemedia.com/papi/media?c=%s" % challenge, imgtype="gif") -- cgit v1.2.3 From 05e11816ea7b6a2cc8d02e6da34d9a352c2431be Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Sep 2014 22:59:52 +0200 Subject: Better dead plugin fallback --- module/plugins/internal/DeadCrypter.py | 7 ++++--- module/plugins/internal/DeadHoster.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 296de739d..1e6fb7d9f 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -6,14 +6,15 @@ from module.plugins.Crypter import Crypter as _Crypter class DeadCrypter(_Crypter): __name__ = "DeadCrypter" __type__ = "crypter" - __version__ = "0.01" + __version__ = "0.02" __pattern__ = None - __description__ = """Crypter is no longer available""" + __description__ = """ Crypter is no longer available """ __author_name__ = "stickell" __author_mail__ = "l.stickell@yahoo.it" def setup(self): - self.fail("Crypter is no longer available") + self.pyfile.error = "Crypter is no longer available" + self.offline() #@TODO: self.offline("Crypter is no longer available") diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index 35a948824..3fa5190d7 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -14,14 +14,15 @@ def create_getInfo(plugin): class DeadHoster(_Hoster): __name__ = "DeadHoster" __type__ = "hoster" - __version__ = "0.11" + __version__ = "0.12" __pattern__ = None - __description__ = """Hoster is no longer available""" + __description__ = """ Hoster is no longer available """ __author_name__ = "zoidberg" __author_mail__ = "zoidberg@mujmail.cz" def setup(self): - self.fail("Hoster is no longer available") + self.pyfile.error = "Hoster is no longer available" + self.offline() #@TODO: self.offline("Hoster is no longer available") -- cgit v1.2.3 From 0d220d634e512d89bda540f91c643b361c82ea8a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 27 Sep 2014 01:38:32 +0200 Subject: Logging string cosmetics --- module/plugins/internal/MultiHoster.py | 16 ++++++++-------- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 9 ++++----- 3 files changed, 13 insertions(+), 14 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index f6d202ee9..7f1bb5103 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -40,7 +40,7 @@ class MultiHoster(Hook): try: hosterSet = self.toHosterSet(self.getHoster()) - set(self.ignored) except Exception, e: - self.logError("%s" % str(e)) + self.logError(e) return [] try: @@ -54,7 +54,7 @@ class MultiHoster(Hook): hosterSet -= configSet except Exception, e: - self.logError("%s" % str(e)) + self.logError(e) self.hosters = list(hosterSet) @@ -98,7 +98,7 @@ class MultiHoster(Hook): def periodical(self): """reload hoster list periodically""" - self.logInfo("Reloading supported hoster list") + self.logInfo(_("Reloading supported hoster list")) old_supported = self.supported self.supported, self.new_supported, self.hosters = [], [], [] @@ -107,7 +107,7 @@ class MultiHoster(Hook): old_supported = [hoster for hoster in old_supported if hoster not in self.supported] if old_supported: - self.logDebug("UNLOAD: %s" % ", ".join(old_supported)) + self.logDebug("UNLOAD", ", ".join(old_supported)) for hoster in old_supported: self.unloadHoster(hoster) @@ -138,24 +138,24 @@ class MultiHoster(Hook): klass = getattr(module, self.__name__) # inject plugin plugin - self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) + self.logDebug("Overwritten Hosters", ", ".join(sorted(self.supported))) for hoster in self.supported: dict = self.core.pluginManager.hosterPlugins[hoster] dict['new_module'] = module dict['new_name'] = self.__name__ if excludedList: - self.logInfo("The following hosters were not overwritten - account exists: %s" % ", ".join(sorted(excludedList))) + self.logInfo(_("The following hosters were not overwritten - account exists"), ", ".join(sorted(excludedList))) if self.new_supported: - self.logDebug("New Hosters: %s" % ", ".join(sorted(self.new_supported))) + self.logDebug("New Hosters", ", ".join(sorted(self.new_supported))) # create new regexp regexp = r".*(%s).*" % "|".join([x.replace(".", "\\.") for x in self.new_supported]) if hasattr(klass, "__pattern__") and isinstance(klass.__pattern__, basestring) and '://' in klass.__pattern__: regexp = r"%s|%s" % (klass.__pattern__, regexp) - self.logDebug("Regexp: %s" % regexp) + self.logDebug("Regexp", regexp) dict = self.core.pluginManager.hosterPlugins[self.__name__] dict['pattern'] = regexp diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 184ba66c9..a4134dcc0 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -83,7 +83,7 @@ class SimpleCrypter(Crypter): if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): self.handleMultiPages() - self.logDebug('Package has %d links' % len(self.package_links)) + self.logDebug("Package has %d links" % len(self.package_links)) if self.package_links: self.packages = [(package_name, self.package_links, folder_name)] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6ecb2d9f2..c18bfff15 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -15,7 +15,6 @@ def replace_patterns(string, ruleslist): for r in ruleslist: rf, rt = r string = re.sub(rf, rt, string) - #self.logDebug(rf, rt, string) return string @@ -230,7 +229,7 @@ class SimpleHoster(Hoster): def getFileInfo(self): - self.logDebug("URL: %s" % self.pyfile.url) + self.logDebug("URL", self.pyfile.url) name, size, status = parseFileInfo(self)[:3] @@ -250,7 +249,7 @@ class SimpleHoster(Hoster): if size: self.pyfile.size = size else: - self.logError("File size not parsed") + self.logError(_("File size not parsed")) self.logDebug("FILE NAME: %s FILE SIZE: %s" % (self.pyfile.name, self.pyfile.size)) return self.file_info @@ -276,7 +275,7 @@ class SimpleHoster(Hoster): time_str = "(unknown time)" max_tries = 100 - self.logInfo("Download limit reached, reconnect or wait %s" % time_str) + self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str) self.setWait(wait_time, True) self.wait() @@ -292,7 +291,7 @@ class SimpleHoster(Hoster): if traffic == -1: return True size = self.pyfile.size / 1024 - self.logInfo("Filesize: %i KiB, Traffic left for user %s: %i KiB" % (size, self.user, traffic)) + self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) return size <= traffic -- cgit v1.2.3 From 1b65270c9713a8291b4ea5e5c208790678cf4e36 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 27 Sep 2014 03:17:16 +0200 Subject: [UnRar] Fix import error --- module/plugins/internal/UnRar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 91be69433..c93b18435 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -4,7 +4,7 @@ import os import re from glob import glob -from os import basename, join +from os.path import basename, join from string import digits from subprocess import Popen, PIPE @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(AbtractExtractor): __name__ = "UnRar" - __version__ = "0.17" + __version__ = "0.18" __description__ = """Rar extractor plugin""" __author_name__ = "RaNaN" -- cgit v1.2.3 From 8add4efc7dc3eeaa873030930e5bd31d6bdf8126 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 28 Sep 2014 14:57:02 +0200 Subject: [XFileSharingPro] COOKIES preset to english + improved setup routine --- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index a4134dcc0..3c783bcab 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -72,7 +72,7 @@ class SimpleCrypter(Crypter): pyfile.url = replace_patterns(pyfile.url, self.URL_REPLACEMENTS) - self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) + self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING) self.checkOnline() diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index c18bfff15..ecbeec979 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -217,17 +217,10 @@ class SimpleHoster(Hoster): self.fail("This link require a premium account") else: # This line is required due to the getURL workaround. Can be removed in 0.4.10 - self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) + self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING) self.handleFree() - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): - if type(url) == unicode: - url = url.encode('utf8') - return Hoster.load(self, url=url, get=get, post=post, ref=ref, cookies=cookies, - just_header=just_header, decode=decode) - - def getFileInfo(self): self.logDebug("URL", self.pyfile.url) -- cgit v1.2.3 From 2e4101c6b373867d556a410901efb8c85c56593f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 28 Sep 2014 16:27:29 +0200 Subject: Prepare method improves setup one --- module/plugins/internal/SimpleCrypter.py | 16 +++++++++------- module/plugins/internal/SimpleHoster.py | 14 ++++++++++---- 2 files changed, 19 insertions(+), 11 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 3c783bcab..2873a7fa7 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.utils import html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.11" + __version__ = "0.12" __pattern__ = None @@ -58,18 +58,20 @@ class SimpleCrypter(Crypter): LOGIN_PREMIUM = False - def setup(self): - if isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) - - - def decrypt(self, pyfile): + def prepare(self): if self.LOGIN_ACCOUNT and not self.account: self.fail('Required account not found!') if self.LOGIN_PREMIUM and not self.premium: self.fail('Required premium account not found!') + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) + + + def decrypt(self, pyfile): + self.prepare() + pyfile.url = replace_patterns(pyfile.url, self.URL_REPLACEMENTS) self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ecbeec979..1c0916541 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -153,13 +153,13 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.35" + __version__ = "0.36" __pattern__ = None __description__ = """Simple hoster plugin""" - __author_name__ = ("zoidberg", "stickell") - __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it") + __author_name__ = ("zoidberg", "stickell", "Walter Purcaro") + __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it", "vuolter@gmail.com") """ Following patterns should be defined by each hoster: @@ -197,13 +197,19 @@ class SimpleHoster(Hoster): def setup(self): self.resumeDownload = self.multiDL = self.premium + + + def prepare(self): if isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) + self.req.setOption("timeout", 120) def process(self, pyfile): + self.prepare() + pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS) - self.req.setOption("timeout", 120) + # Due to a 0.4.9 core bug self.load would keep previous cookies even if overridden by cookies parameter. # Workaround using getURL. Can be reverted in 0.4.10 as the cookies bug has been fixed. self.html = getURL(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) -- cgit v1.2.3 From 3177003f188d8f48ecafe5483e69f8b838581628 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 28 Sep 2014 16:30:24 +0200 Subject: [DeadHoster] Improve getInfo + two hosters mark dead --- module/plugins/internal/DeadHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index 3fa5190d7..37cd3cf17 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -6,7 +6,7 @@ from module.plugins.Hoster import Hoster as _Hoster def create_getInfo(plugin): def getInfo(urls): - yield [('#N/A: ' + url, 0, 1, url) for url in urls] + yield map(lambda url: ('#N/A: ' + url, 0, 1, url), urls) return getInfo -- cgit v1.2.3 From 37e66401e68c082900ff98da6e38ca092a024933 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 3 Oct 2014 09:08:53 +0200 Subject: [CaptchaService] Improved --- module/plugins/internal/CaptchaService.py | 169 +++++++++++++++++++++++------- 1 file changed, 134 insertions(+), 35 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 26482379d..72a8f8789 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -7,96 +7,195 @@ from random import random class CaptchaService: __name__ = "CaptchaService" - __version__ = "0.06" + __version__ = "0.07" - __description__ = """Captcha service plugin""" + __description__ = """Base captcha service plugin""" __author_name__ = "pyLoad Team" __author_mail__ = "admin@pyload.org" + KEY_PATTERN = None + + key = None + def __init__(self, plugin): self.plugin = plugin -class ReCaptcha: - RECAPTCHA_KEY_PATTERN = r"https?://(?:www\.)?google\.com/recaptcha/api/challenge\?k=(?P\w+)" - RECAPTCHA_KEY_AJAX_PATTERN = r"Recaptcha\.create\s*\(\s*[\"'](?P\w+)[\"']\s*," + def detect_key(self, html=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = "%s html missing" % self.__name__ + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + m = re.search(self.KEY_PATTERN, html) + if m: + self.key = m.group("KEY") + self.plugin.logDebug("%s key: %s" % (self.__name__, self.key) + return self.key + else: + self.plugin.logDebug("%s key not found" % self.__name__) + return None + + + def challenge(self, key): + raise NotImplementedError - recaptcha_key = None + def result(self, server, challenge): + raise NotImplementedError - def __init__(self, plugin): - self.plugin = plugin +class ReCaptcha(CaptchaService): + __name__ = "ReCaptcha" + __version__ = "0.02" - def detect_key(self, html): - m = re.search(self.RECAPTCHA_KEY_PATTERN, html) + __description__ = """ReCaptcha captcha service plugin""" + __author_name__ = "pyLoad Team" + __author_mail__ = "admin@pyload.org" + + + KEY_PATTERN = r"https?://(?:www\.)?google\.com/recaptcha/api/challenge\?k=(?P\w+?)" + KEY_AJAX_PATTERN = r"Recaptcha\.create\s*\(\s*[\"'](?P\w+)[\"']\s*," + + + def detect_key(self, html=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = "ReCaptcha html missing" + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + m = re.search(self.KEY_PATTERN, html) if m is None: - m = re.search(self.RECAPTCHA_KEY_AJAX_PATTERN, html) + m = re.search(self.KEY_AJAX_PATTERN, html) if m: - self.recaptcha_key = m.group('key') - return self.recaptcha_key + self.key = m.group("KEY") + self.plugin.logDebug("ReCaptcha key: %s" % self.key) + return self.key else: + self.plugin.logDebug("ReCaptcha key not found") return None - def challenge(self, key=None): + def challenge(self, key=key): if not key: - if self.recaptcha_key: - key = self.recaptcha_key - else: - raise TypeError("ReCaptcha key not found") + errmsg = "ReCaptcha key missing" + self.plugin.fail(errmsg) + raise TypeError(errmsg) - js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={"k": key}, cookies=True) + js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}, cookies=True) try: - challenge = re.search("challenge : '(.*?)',", js).group(1) - server = re.search("server : '(.*?)',", js).group(1) + challenge = re.search("challenge : '(.+?)',", js).group(1) + server = re.search("server : '(.+?)',", js).group(1) except: - self.plugin.fail("recaptcha error") + self.plugin.parseError("ReCaptcha challenge pattern not found") + result = self.result(server, challenge) return challenge, result def result(self, server, challenge): - return self.plugin.decryptCaptcha("%simage" % server, get={"c": challenge}, + return self.plugin.decryptCaptcha("%simage" % server, get={'c': challenge}, cookies=True, forceUser=True, imgtype="jpg") class AdsCaptcha(CaptchaService): + __name__ = "AdsCaptcha" + __version__ = "0.02" - def challenge(self, src): - js = self.plugin.req.load(src, cookies=True) + __description__ = """AdsCaptcha captcha service plugin""" + __author_name__ = "pyLoad Team" + __author_mail__ = "admin@pyload.org" + + + ID_PATTERN = r'http://api\.adscaptcha\.com/Get\.aspx\?[^"\']*CaptchaId=(?P\d+)' + KEY_PATTERN = r'http://api\.adscaptcha\.com/Get\.aspx\?[^"\']*PublicKey=(?P[\w-]+)' + + + def detect_key(self, html=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = "AdsCaptcha html missing" + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + m = re.search(self.ID_PATTERN, html) + n = re.search(self.KEY_PATTERN, html) + if m and n: + self.key = (m.group("ID"), m.group("KEY")) + self.plugin.logDebug("AdsCaptcha id|key: %s | %s" % self.key) + return self.key + else: + self.plugin.logDebug("AdsCaptcha id or key not found") + return None + + + def challenge(self, key=key): #: key is tuple(CaptchaId, PublicKey) + CaptchaId, PublicKey = key + + if not CaptchaId or not PublicKey: + errmsg = "AdsCaptcha key missing" + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + js = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}, cookies=True) try: - challenge = re.search("challenge: '(.*?)',", js).group(1) - server = re.search("server: '(.*?)',", js).group(1) + challenge = re.search("challenge: '(.+?)',", js).group(1) + server = re.search("server: '(.+?)',", js).group(1) except: - self.plugin.fail("adscaptcha error") + self.plugin.parseError("AdsCaptcha challenge pattern not found") + result = self.result(server, challenge) return challenge, result def result(self, server, challenge): - return self.plugin.decryptCaptcha("%sChallenge.aspx" % server, get={"cid": challenge, "dummy": random()}, + return self.plugin.decryptCaptcha("%sChallenge.aspx" % server, get={'cid': challenge, 'dummy': random()}, cookies=True, imgtype="jpg") class SolveMedia(CaptchaService): + __name__ = "SolveMedia" + __version__ = "0.02" - def challenge(self, src): - html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript?k=%s" % src, cookies=True) + __description__ = """SolveMedia captcha service plugin""" + __author_name__ = "pyLoad Team" + __author_mail__ = "admin@pyload.org" + + + KEY_PATTERN = r'http://api\.solvemedia\.com/papi/challenge\.(no)?script\?k=(?P.+?)"' + + + def challenge(self, key=key): + if not key: + errmsg = "SolveMedia key missing" + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript", get={'k': key}, cookies=True) try: challenge = re.search(r'', html).group(1) + server = "http://api.solvemedia.com/papi/media" except: - self.plugin.fail("solvemedia error") - result = self.result(challenge) + self.plugin.parseError("SolveMedia challenge pattern not found") + + result = self.result(server, challenge) return challenge, result - def result(self, challenge): - return self.plugin.decryptCaptcha("http://api.solvemedia.com/papi/media?c=%s" % challenge, imgtype="gif") + def result(self, server, challenge): + return self.plugin.decryptCaptcha(server, get={'c': challenge}, imgtype="gif") -- cgit v1.2.3 From 23ae563604dca1dae262fbc598154b99b2f1eae8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 3 Oct 2014 09:06:05 +0200 Subject: Update plugins after CaptchaService and XFileSharingPro changes --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 1c0916541..3625f171e 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -176,7 +176,7 @@ class SimpleHoster(Hoster): example: OFFLINE_PATTERN = r'File (deleted|not found)' TEMP_OFFLINE_PATTERN: Checks if the file is temporarily offline - example: TEMP_OFFLINE_PATTERN = r'Server maintainance' + example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account example: PREMIUM_ONLY_PATTERN = r'Premium account required' -- cgit v1.2.3 From 7b3e630f60ab2a917106cc6e09b92123001a0083 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 3 Oct 2014 09:25:46 +0200 Subject: [CaptchaService] Fix code indentation typo --- module/plugins/internal/CaptchaService.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 72a8f8789..caac14d61 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -7,7 +7,7 @@ from random import random class CaptchaService: __name__ = "CaptchaService" - __version__ = "0.07" + __version__ = "0.08" __description__ = """Base captcha service plugin""" __author_name__ = "pyLoad Team" @@ -83,7 +83,7 @@ class ReCaptcha(CaptchaService): return None - def challenge(self, key=key): + def challenge(self, key=key): if not key: errmsg = "ReCaptcha key missing" self.plugin.fail(errmsg) -- cgit v1.2.3 From 8e9afb4c9c83a2a46cc491fe9f48d22284759bf0 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 3 Oct 2014 10:52:14 +0200 Subject: [CaptchaService] Fix typo --- module/plugins/internal/CaptchaService.py | 41 +++++++++++++++++++------------ 1 file changed, 25 insertions(+), 16 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index caac14d61..45e56ba42 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -7,7 +7,7 @@ from random import random class CaptchaService: __name__ = "CaptchaService" - __version__ = "0.08" + __version__ = "0.09" __description__ = """Base captcha service plugin""" __author_name__ = "pyLoad Team" @@ -34,7 +34,7 @@ class CaptchaService: m = re.search(self.KEY_PATTERN, html) if m: self.key = m.group("KEY") - self.plugin.logDebug("%s key: %s" % (self.__name__, self.key) + self.plugin.logDebug("%s key: %s" % (self.__name__, self.key)) return self.key else: self.plugin.logDebug("%s key not found" % self.__name__) @@ -83,11 +83,14 @@ class ReCaptcha(CaptchaService): return None - def challenge(self, key=key): + def challenge(self, key): if not key: - errmsg = "ReCaptcha key missing" - self.plugin.fail(errmsg) - raise TypeError(errmsg) + if self.key: + key = self.key + else: + errmsg = "ReCaptcha key missing" + self.plugin.fail(errmsg) + raise TypeError(errmsg) js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}, cookies=True) @@ -140,13 +143,16 @@ class AdsCaptcha(CaptchaService): return None - def challenge(self, key=key): #: key is tuple(CaptchaId, PublicKey) - CaptchaId, PublicKey = key + def challenge(self, key): #: key is tuple(CaptchaId, PublicKey) + if not key: + if self.key: + key = self.key + else: + errmsg = "AdsCaptcha key missing" + self.plugin.fail(errmsg) + raise TypeError(errmsg) - if not CaptchaId or not PublicKey: - errmsg = "AdsCaptcha key missing" - self.plugin.fail(errmsg) - raise TypeError(errmsg) + CaptchaId, PublicKey = key js = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}, cookies=True) @@ -178,11 +184,14 @@ class SolveMedia(CaptchaService): KEY_PATTERN = r'http://api\.solvemedia\.com/papi/challenge\.(no)?script\?k=(?P.+?)"' - def challenge(self, key=key): + def challenge(self, key): if not key: - errmsg = "SolveMedia key missing" - self.plugin.fail(errmsg) - raise TypeError(errmsg) + if self.key: + key = self.key + else: + errmsg = "SolveMedia key missing" + self.plugin.fail(errmsg) + raise TypeError(errmsg) html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript", get={'k': key}, cookies=True) try: -- cgit v1.2.3 From 0d4b92d58e6d4e959aeaa780cf29b8d88f46610a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 3 Oct 2014 14:00:46 +0200 Subject: Spare code cosmetics --- module/plugins/internal/CaptchaService.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 45e56ba42..b2fba0652 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -41,7 +41,7 @@ class CaptchaService: return None - def challenge(self, key): + def challenge(self, key=None): raise NotImplementedError @@ -83,7 +83,7 @@ class ReCaptcha(CaptchaService): return None - def challenge(self, key): + def challenge(self, key=None): if not key: if self.key: key = self.key @@ -143,7 +143,7 @@ class AdsCaptcha(CaptchaService): return None - def challenge(self, key): #: key is tuple(CaptchaId, PublicKey) + def challenge(self, key=None): #: key is tuple(CaptchaId, PublicKey) if not key: if self.key: key = self.key @@ -184,7 +184,7 @@ class SolveMedia(CaptchaService): KEY_PATTERN = r'http://api\.solvemedia\.com/papi/challenge\.(no)?script\?k=(?P.+?)"' - def challenge(self, key): + def challenge(self, key=None): if not key: if self.key: key = self.key -- cgit v1.2.3 From 1047c706d41dec4a49a747496c2ec01ed3bf7801 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 5 Oct 2014 01:54:58 +0200 Subject: Use load instead req.load in all crypters and CaptchaService --- module/plugins/internal/CaptchaService.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index b2fba0652..35074bc63 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -92,7 +92,7 @@ class ReCaptcha(CaptchaService): self.plugin.fail(errmsg) raise TypeError(errmsg) - js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}, cookies=True) + js = self.plugin.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}, cookies=True) try: challenge = re.search("challenge : '(.+?)',", js).group(1) @@ -154,7 +154,7 @@ class AdsCaptcha(CaptchaService): CaptchaId, PublicKey = key - js = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}, cookies=True) + js = self.plugin.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}, cookies=True) try: challenge = re.search("challenge: '(.+?)',", js).group(1) @@ -193,7 +193,7 @@ class SolveMedia(CaptchaService): self.plugin.fail(errmsg) raise TypeError(errmsg) - html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript", get={'k': key}, cookies=True) + html = self.plugin.load("http://api.solvemedia.com/papi/challenge.noscript", get={'k': key}, cookies=True) try: challenge = re.search(r'', html).group(1) -- cgit v1.2.3 From 56971d70884e7d4f3242e7c52da88c4298ba130a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 5 Oct 2014 02:26:08 +0200 Subject: [XFSPAccount] COOKIES attribute support --- module/plugins/internal/XFSPAccount.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index 5c0bfc893..b93fb5d30 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -5,20 +5,23 @@ import re from time import mktime, strptime from module.plugins.Account import Account -from module.plugins.internal.SimpleHoster import parseHtmlForm +from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies from module.utils import parseFileSize class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.06" + __version__ = "0.07" __description__ = """XFileSharingPro base account plugin""" __author_name__ = "zoidberg" __author_mail__ = "zoidberg@mujmail.cz" - MAIN_PAGE = None + + HOSTER_URL = None + + COOKIES = None #: or list of tuples [(domain, name, value)] VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:([^<]+)' TRAFFIC_LEFT_PATTERN = r'>Traffic available today:([^<]+)' @@ -27,7 +30,7 @@ class XFSPAccount(Account): def loadAccountInfo(self, user, req): - html = req.load(self.MAIN_PAGE + "?op=my_account", decode=True) + html = req.load(self.HOSTER_URL + "?op=my_account", decode=True) validuntil = trafficleft = None premium = True if re.search(self.PREMIUM_PATTERN, html) else False @@ -52,18 +55,21 @@ class XFSPAccount(Account): return {"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium} + def login(self, user, data, req): - html = req.load('%slogin.html' % self.MAIN_PAGE, decode=True) + set_cookies(req.cj, self.COOKIES) + + html = req.load('%slogin.html' % self.HOSTER_URL, decode=True) action, inputs = parseHtmlForm('name="FL"', html) if not inputs: inputs = {"op": "login", - "redirect": self.MAIN_PAGE} + "redirect": self.HOSTER_URL} inputs.update({"login": user, "password": data['password']}) - html = req.load(self.MAIN_PAGE, post=inputs, decode=True) + html = req.load(self.HOSTER_URL, post=inputs, decode=True) if re.search(self.LOGIN_FAIL_PATTERN, html): self.wrongPassword() -- cgit v1.2.3 From b994d8fa6a7dcc902787bb2450acdbed535d72fc Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 5 Oct 2014 22:11:49 +0200 Subject: [SimpleHoster] Added default handle patterns LINK_FREE_PATTERN and LINK_PREMIUM_PATTERN --- module/plugins/internal/SimpleHoster.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3625f171e..160125cdf 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -153,7 +153,7 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.36" + __version__ = "0.37" __pattern__ = None @@ -255,11 +255,35 @@ class SimpleHoster(Hoster): def handleFree(self): - self.fail("Free download not implemented") + if not hasattr(self, 'LINK_FREE_PATTERN'): + self.fail("Free download not implemented") + + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m is None: + self.parseError("Free download link not found") + + try: + link = m.group(1) + except Exception, e: + self.logError(str(e)) + else: + self.download(link, ref=True, cookies=True, disposition=True) def handlePremium(self): - self.fail("Premium download not implemented") + if not hasattr(self, 'LINK_PREMIUM_PATTERN'): + self.fail("Premium download not implemented") + + m = re.search(self.LINK_PREMIUM_PATTERN, self.html) + if m is None: + self.parseError("Premium download link not found") + + try: + link = m.group(1) + except Exception, e: + self.logError(str(e)) + else: + self.download(link, ref=True, cookies=True, disposition=True) def parseError(self, msg): -- cgit v1.2.3 From ec72263da9135c57e628bbaad4aa570f3df02fe0 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 6 Oct 2014 13:23:26 +0200 Subject: [XFSPAccount] Improve patterns --- module/plugins/internal/XFSPAccount.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index b93fb5d30..434bf8d22 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -2,6 +2,7 @@ import re +from urlparse import urljoin from time import mktime, strptime from module.plugins.Account import Account @@ -12,25 +13,25 @@ from module.utils import parseFileSize class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.07" + __version__ = "0.08" __description__ = """XFileSharingPro base account plugin""" - __author_name__ = "zoidberg" - __author_mail__ = "zoidberg@mujmail.cz" + __author_name__ = ("zoidberg", "Walter Purcaro") + __author_mail__ = ("zoidberg@mujmail.cz", "vuolter@gmail.com") HOSTER_URL = None COOKIES = None #: or list of tuples [(domain, name, value)] - VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:([^<]+)' - TRAFFIC_LEFT_PATTERN = r'>Traffic available today:([^<]+)' - LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|>Error<' + VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(.+?)' + TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?(.+?)' + LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' PREMIUM_PATTERN = r'>Renew premium<' def loadAccountInfo(self, user, req): - html = req.load(self.HOSTER_URL + "?op=my_account", decode=True) + html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) validuntil = trafficleft = None premium = True if re.search(self.PREMIUM_PATTERN, html) else False @@ -53,21 +54,22 @@ class XFSPAccount(Account): else: trafficleft = parseFileSize(trafficleft) / 1024 - return {"validuntil": validuntil, "trafficleft": trafficleft, "premium": premium} + return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} def login(self, user, data, req): set_cookies(req.cj, self.COOKIES) - html = req.load('%slogin.html' % self.HOSTER_URL, decode=True) + url = urljoin(self.HOSTER_URL, "login.html") + html = req.load(url, decode=True) action, inputs = parseHtmlForm('name="FL"', html) if not inputs: - inputs = {"op": "login", - "redirect": self.HOSTER_URL} + inputs = {'op': "login", + 'redirect': self.HOSTER_URL} - inputs.update({"login": user, - "password": data['password']}) + inputs.update({'login': user, + 'password': data['password']}) html = req.load(self.HOSTER_URL, post=inputs, decode=True) -- cgit v1.2.3 From 6ec5c8215a774455b405db28ad9d936a5ee60215 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 6 Oct 2014 13:26:06 +0200 Subject: [CaptchaService] Restore req.load --- module/plugins/internal/CaptchaService.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 35074bc63..b2fba0652 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -92,7 +92,7 @@ class ReCaptcha(CaptchaService): self.plugin.fail(errmsg) raise TypeError(errmsg) - js = self.plugin.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}, cookies=True) + js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}, cookies=True) try: challenge = re.search("challenge : '(.+?)',", js).group(1) @@ -154,7 +154,7 @@ class AdsCaptcha(CaptchaService): CaptchaId, PublicKey = key - js = self.plugin.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}, cookies=True) + js = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}, cookies=True) try: challenge = re.search("challenge: '(.+?)',", js).group(1) @@ -193,7 +193,7 @@ class SolveMedia(CaptchaService): self.plugin.fail(errmsg) raise TypeError(errmsg) - html = self.plugin.load("http://api.solvemedia.com/papi/challenge.noscript", get={'k': key}, cookies=True) + html = self.plugin.req.load("http://api.solvemedia.com/papi/challenge.noscript", get={'k': key}, cookies=True) try: challenge = re.search(r'', html).group(1) -- cgit v1.2.3 From 82facc20b829d182bd0ae5787d29ed21f0f93fbc Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 6 Oct 2014 14:28:32 +0200 Subject: [SimpleHoster] Better filename processing + info about new pattern attributes --- module/plugins/internal/SimpleHoster.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 160125cdf..6ad8398bb 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -106,7 +106,7 @@ def parseFileInfo(self, url='', html=''): # File online, return name and size info['status'] = 2 if 'N' in info: - info['name'] = replace_patterns(info['N'], self.FILE_NAME_REPLACEMENTS) + info['name'] = replace_patterns(info['N'].strip(), self.FILE_NAME_REPLACEMENTS) if 'S' in info: size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], self.FILE_SIZE_REPLACEMENTS) @@ -153,7 +153,7 @@ class PluginParseError(Exception): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.37" + __version__ = "0.38" __pattern__ = None @@ -180,6 +180,12 @@ class SimpleHoster(Hoster): PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account example: PREMIUM_ONLY_PATTERN = r'Premium account required' + + + Instead overriding handleFree and handlePremium methods now you can define patterns for direct download: + + LINK_FREE_PATTERN: (optional) Get direct link for free download + LINK_PREMIUM_PATTERN: (optional) Get direct link for premium download """ FILE_NAME_REPLACEMENTS = [("&#?\w+;", fixup)] -- cgit v1.2.3 From cf57dbf9a57bd4078051688f8984a4c1fe6e4996 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 6 Oct 2014 14:45:15 +0200 Subject: [SimpleCrypter] Improve patterns --- module/plugins/internal/SimpleCrypter.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 2873a7fa7..d69995402 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -4,13 +4,13 @@ import re from module.plugins.Crypter import Crypter from module.plugins.internal.SimpleHoster import PluginParseError, replace_patterns, set_cookies -from module.utils import html_unescape +from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.12" + __version__ = "0.13" __pattern__ = None @@ -24,8 +24,8 @@ class SimpleCrypter(Crypter): LINK_PATTERN: group(1) must be a download link or a regex to catch more links example: LINK_PATTERN = r'
Date: Tue, 7 Oct 2014 13:09:15 +0200 Subject: [XFSPAccount] Improve loadAccountInfo --- module/plugins/internal/XFSPAccount.py | 49 +++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 15 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index 434bf8d22..f8440b0b2 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -13,7 +13,7 @@ from module.utils import parseFileSize class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.08" + __version__ = "0.09" __description__ = """XFileSharingPro base account plugin""" __author_name__ = ("zoidberg", "Walter Purcaro") @@ -25,34 +25,53 @@ class XFSPAccount(Account): COOKIES = None #: or list of tuples [(domain, name, value)] VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(.+?)' - TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?(.+?)' + TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?(?P.+?)' LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' - PREMIUM_PATTERN = r'>Renew premium<' + # PREMIUM_PATTERN = r'>Renew premium<' def loadAccountInfo(self, user, req): html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) - validuntil = trafficleft = None - premium = True if re.search(self.PREMIUM_PATTERN, html) else False + validuntil = None + trafficleft = None + premium = None + + if hasattr(self, "PREMIUM_PATTERN"): + premium = True if re.search(self.PREMIUM_PATTERN, html) else False m = re.search(self.VALID_UNTIL_PATTERN, html) if m: - premium = True - trafficleft = -1 + expiredate = m.group(1) + self.logDebug("Expire date: " + expiredate) + try: - self.logDebug(m.group(1)) - validuntil = mktime(strptime(m.group(1), "%d %B %Y")) + validuntil = mktime(strptime(expiredate, "%d %B %Y")) except Exception, e: self.logError(e) - else: - m = re.search(self.TRAFFIC_LEFT_PATTERN, html) - if m: - trafficleft = m.group(1) - if "Unlimited" in trafficleft: + else: + if validuntil > mktime(gmtime()): premium = True + trafficleft = -1 else: - trafficleft = parseFileSize(trafficleft) / 1024 + if premium is False: #: registered account type (not premium) + validuntil = -1 + premium = False + + try: + traffic = re.search(self.TRAFFIC_LEFT_PATTERN, html).groupdict() + trafficsize = traffic['S'] + traffic['U'] if 'U' in traffic else traffic['S'] + if "Unlimited" in trafficsize: + trafficleft = -1 + if premium is None: + premium = True + else: + trafficleft = parseFileSize(trafficsize) + except: + pass + + if premium is None: + premium = False return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} -- cgit v1.2.3 From dd24571c5015e3fea1877671b464104557463f6f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 7 Oct 2014 13:52:47 +0200 Subject: [SimpleCrypter][SimpleHoster] Better exception handling --- module/plugins/internal/SimpleCrypter.py | 20 ++++++++++++-------- module/plugins/internal/SimpleHoster.py | 29 ++++++++++++++++------------- 2 files changed, 28 insertions(+), 21 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index d69995402..43e8b46cb 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -18,11 +18,12 @@ class SimpleCrypter(Crypter): __author_name__ = ("stickell", "zoidberg", "Walter Purcaro") __author_mail__ = ("l.stickell@yahoo.it", "zoidberg@mujmail.cz", "vuolter@gmail.com") + """ Following patterns should be defined by each crypter: LINK_PATTERN: group(1) must be a download link or a regex to catch more links - example: LINK_PATTERN = r'
Files of: ([^<]+) folder' @@ -48,7 +49,6 @@ class SimpleCrypter(Crypter): return the html of the page number page_n """ - TITLE_REPLACEMENTS = [("&#?\w+;", fixup)] URL_REPLACEMENTS = [] @@ -111,24 +111,28 @@ class SimpleCrypter(Crypter): def getPackageNameAndFolder(self): if hasattr(self, 'TITLE_PATTERN'): - m = re.search(self.TITLE_PATTERN, self.html) - if m: + try: + m = re.search(self.TITLE_PATTERN, self.html) name = replace_patterns(m.group(1).strip(), self.TITLE_REPLACEMENTS) folder = html_unescape(name) + except: + pass + else: self.logDebug("Found name [%s] and folder [%s] in package info" % (name, folder)) return name, folder name = self.pyfile.package().name folder = self.pyfile.package().folder self.logDebug("Package info not found, defaulting to pyfile name [%s] and folder [%s]" % (name, folder)) + return name, folder def handleMultiPages(self): - pages = re.search(self.PAGES_PATTERN, self.html) - if pages: - pages = int(pages.group(1)) - else: + try: + m = re.search(self.PAGES_PATTERN, self.html) + pages = int(m.group(1)) + except: pages = 1 for p in xrange(2, pages + 1): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6ad8398bb..238c98b05 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -112,9 +112,8 @@ def parseFileInfo(self, url='', html=''): self.FILE_SIZE_REPLACEMENTS) info['size'] = parseFileSize(size) elif isinstance(info['size'], basestring): - if 'units' in info: - info['size'] += info['units'] - info['size'] = parseFileSize(info['size']) + unit = info['units'] if 'units' in info else None + info['size'] = parseFileSize(info['size'], unit) if hasattr(self, "file_info"): self.file_info = info @@ -161,6 +160,7 @@ class SimpleHoster(Hoster): __author_name__ = ("zoidberg", "stickell", "Walter Purcaro") __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it", "vuolter@gmail.com") + """ Following patterns should be defined by each hoster: @@ -184,8 +184,11 @@ class SimpleHoster(Hoster): Instead overriding handleFree and handlePremium methods now you can define patterns for direct download: - LINK_FREE_PATTERN: (optional) Get direct link for free download - LINK_PREMIUM_PATTERN: (optional) Get direct link for premium download + LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download + example: LINK_FREE_PATTERN = r'
Date: Tue, 7 Oct 2014 18:57:59 +0200 Subject: New __authors__ key replaces __author_name__ and __author_mail__ + Whitespaces and EOF fixup --- module/plugins/internal/AbstractExtractor.py | 3 +-- module/plugins/internal/CaptchaService.py | 13 +++++-------- module/plugins/internal/DeadCrypter.py | 3 +-- module/plugins/internal/DeadHoster.py | 3 +-- module/plugins/internal/MultiHoster.py | 4 ++-- module/plugins/internal/SimpleCrypter.py | 5 +++-- module/plugins/internal/SimpleHoster.py | 5 +++-- module/plugins/internal/UnRar.py | 3 +-- module/plugins/internal/UnZip.py | 3 +-- module/plugins/internal/XFSPAccount.py | 4 ++-- 10 files changed, 20 insertions(+), 26 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index d1d1a09cb..fefeb4ece 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -17,8 +17,7 @@ class AbtractExtractor: __version__ = "0.1" __description__ = """Abtract extractor plugin""" - __author_name__ = "pyLoad Team" - __author_mail__ = "admin@pyload.org" + __authors__ = [("pyLoad Team", "admin@pyload.org")] @staticmethod diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index b2fba0652..dad537bec 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -10,8 +10,8 @@ class CaptchaService: __version__ = "0.09" __description__ = """Base captcha service plugin""" - __author_name__ = "pyLoad Team" - __author_mail__ = "admin@pyload.org" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + KEY_PATTERN = None @@ -54,8 +54,7 @@ class ReCaptcha(CaptchaService): __version__ = "0.02" __description__ = """ReCaptcha captcha service plugin""" - __author_name__ = "pyLoad Team" - __author_mail__ = "admin@pyload.org" + __authors__ = [("pyLoad Team", "admin@pyload.org")] KEY_PATTERN = r"https?://(?:www\.)?google\.com/recaptcha/api/challenge\?k=(?P\w+?)" @@ -115,8 +114,7 @@ class AdsCaptcha(CaptchaService): __version__ = "0.02" __description__ = """AdsCaptcha captcha service plugin""" - __author_name__ = "pyLoad Team" - __author_mail__ = "admin@pyload.org" + __authors__ = [("pyLoad Team", "admin@pyload.org")] ID_PATTERN = r'http://api\.adscaptcha\.com/Get\.aspx\?[^"\']*CaptchaId=(?P\d+)' @@ -177,8 +175,7 @@ class SolveMedia(CaptchaService): __version__ = "0.02" __description__ = """SolveMedia captcha service plugin""" - __author_name__ = "pyLoad Team" - __author_mail__ = "admin@pyload.org" + __authors__ = [("pyLoad Team", "admin@pyload.org")] KEY_PATTERN = r'http://api\.solvemedia\.com/papi/challenge\.(no)?script\?k=(?P.+?)"' diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 1e6fb7d9f..8886be0a7 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -11,8 +11,7 @@ class DeadCrypter(_Crypter): __pattern__ = None __description__ = """ Crypter is no longer available """ - __author_name__ = "stickell" - __author_mail__ = "l.stickell@yahoo.it" + __authors__ = [("stickell", "l.stickell@yahoo.it")] def setup(self): diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index 37cd3cf17..fff254de7 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -19,8 +19,7 @@ class DeadHoster(_Hoster): __pattern__ = None __description__ = """ Hoster is no longer available """ - __author_name__ = "zoidberg" - __author_mail__ = "zoidberg@mujmail.cz" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] def setup(self): diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 7f1bb5103..daab1e855 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -11,8 +11,8 @@ class MultiHoster(Hook): __version__ = "0.19" __description__ = """Generic MultiHoster plugin""" - __author_name__ = "pyLoad Team" - __author_mail__ = "admin@pyload.org" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + replacements = [("2shared.com", "twoshared.com"), ("4shared.com", "fourshared.com"), ("cloudnator.com", "shragle.com"), ("ifile.it", "filecloud.io"), ("easy-share.com", "crocko.com"), ("freakshare.net", "freakshare.com"), diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 43e8b46cb..2da551ea9 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -15,8 +15,9 @@ class SimpleCrypter(Crypter): __pattern__ = None __description__ = """Simple decrypter plugin""" - __author_name__ = ("stickell", "zoidberg", "Walter Purcaro") - __author_mail__ = ("l.stickell@yahoo.it", "zoidberg@mujmail.cz", "vuolter@gmail.com") + __authors__ = [("stickell", "l.stickell@yahoo.it"), + ("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] """ diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 238c98b05..3887f1579 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -157,8 +157,9 @@ class SimpleHoster(Hoster): __pattern__ = None __description__ = """Simple hoster plugin""" - __author_name__ = ("zoidberg", "stickell", "Walter Purcaro") - __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it", "vuolter@gmail.com") + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] """ diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index c93b18435..11496b606 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -25,8 +25,7 @@ class UnRar(AbtractExtractor): __version__ = "0.18" __description__ = """Rar extractor plugin""" - __author_name__ = "RaNaN" - __author_mail__ = "RaNaN@pyload.org" + __authors__ = [("RaNaN", "RaNaN@pyload.org")] CMD = "unrar" diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index e339434f9..71f144fc7 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -11,8 +11,7 @@ class UnZip(AbtractExtractor): __version__ = "0.1" __description__ = """Zip extractor plugin""" - __author_name__ = "RaNaN" - __author_mail__ = "RaNaN@pyload.org" + __authors__ = [("RaNaN", "RaNaN@pyload.org")] @staticmethod diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index f8440b0b2..b2fb6179e 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -16,8 +16,8 @@ class XFSPAccount(Account): __version__ = "0.09" __description__ = """XFileSharingPro base account plugin""" - __author_name__ = ("zoidberg", "Walter Purcaro") - __author_mail__ = ("zoidberg@mujmail.cz", "vuolter@gmail.com") + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] HOSTER_URL = None -- cgit v1.2.3 From 0e9ba8a76e88ee98497db8dc7f25d8d160f10fa4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 8 Oct 2014 00:30:48 +0200 Subject: [XFSPAccount] Fix missing COOKIE typecheck --- module/plugins/internal/XFSPAccount.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index b2fb6179e..2cda54615 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -13,7 +13,7 @@ from module.utils import parseFileSize class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.09" + __version__ = "0.10" __description__ = """XFileSharingPro base account plugin""" __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), @@ -77,7 +77,8 @@ class XFSPAccount(Account): def login(self, user, data, req): - set_cookies(req.cj, self.COOKIES) + if isinstance(self.COOKIES, list): + set_cookies(req.cj, self.COOKIES) url = urljoin(self.HOSTER_URL, "login.html") html = req.load(url, decode=True) -- cgit v1.2.3 From b7bc4c7a920cb6e768948d8489af40bf36a72810 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 8 Oct 2014 16:04:14 +0200 Subject: [XFileSharingPro] Rename to XFSPHoster --- module/plugins/internal/XFSPHoster.py | 361 ++++++++++++++++++++++++++++++++++ 1 file changed, 361 insertions(+) create mode 100644 module/plugins/internal/XFSPHoster.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py new file mode 100644 index 000000000..2376c1b84 --- /dev/null +++ b/module/plugins/internal/XFSPHoster.py @@ -0,0 +1,361 @@ +# -*- coding: utf-8 -*- + +import re + +from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME +from random import random +from urllib import unquote +from urlparse import urlparse + +from module.network.RequestFactory import getURL +from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia +from module.plugins.internal.SimpleHoster import create_getInfo, PluginParseError, replace_patterns, set_cookies, SimpleHoster +from module.utils import html_unescape + + +class XFSPHoster(SimpleHoster): + """ + Common base for XFileSharingPro hosters like EasybytezCom, CramitIn, FiledinoCom... + Some hosters may work straight away when added to __pattern__ + However, most of them will NOT work because they are either down or running a customized version + """ + __name__ = "XFSPHoster" + __type__ = "hoster" + __version__ = "0.37" + + __pattern__ = r'^unmatchable$' + + __description__ = """XFileSharingPro base hoster plugin""" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_NAME = None + + FILE_URL_REPLACEMENTS = [(r'/embed-(\w{12}).*', r'/\1')] #: support embedded files + + COOKIES = [(HOSTER_NAME, "lang", "english")] + + FILE_INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' + FILE_NAME_PATTERN = r'[\d\.\,]+) ?(?P\w+)?\)' + + OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' + + WAIT_PATTERN = r'.*?>(\d+)' + + OVR_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' + LINK_PATTERN = None #: final download url pattern + + CAPTCHA_URL_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' + CAPTCHA_DIV_PATTERN = r'>Enter code.*?(.+?)
' + RECAPTCHA_PATTERN = None + SOLVEMEDIA_PATTERN = None + + ERROR_PATTERN = r'class=["\']err["\'][^>]*>(.+?)", " ", self.errmsg)) + + if 'wait' in self.errmsg: + wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in + re.findall(r'(\d+)\s*(hour|minute|second)', self.errmsg)]) + self.wait(wait_time, True) + elif 'captcha' in self.errmsg: + self.invalidCaptcha() + elif 'premium' in self.errmsg and 'require' in self.errmsg: + self.fail("File can be downloaded by premium users only") + elif 'limit' in self.errmsg: + self.wait(1 * 60 * 60, True) + self.retry(25) + elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: + self.retry() + elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: + self.tempOffline() + elif 'download files up to' in self.errmsg: + self.fail("File too large for free download") + else: + self.fail(self.errmsg) + + else: + self.errmsg = None + + return self.errmsg + + + def getPostParameters(self): + for _ in xrange(3): + if not self.errmsg: + self.checkErrors() + + if hasattr(self, "FORM_PATTERN"): + action, inputs = self.parseHtmlForm(self.FORM_PATTERN) + else: + action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) + + if not inputs: + action, inputs = self.parseHtmlForm('F1') + if not inputs: + if self.errmsg: + self.retry() + else: + self.parseError("Form not found") + + self.logDebug(self.HOSTER_NAME, inputs) + + if 'op' in inputs and inputs['op'] in ("download2", "download3"): + if "password" in inputs: + if self.passwords: + inputs['password'] = self.passwords.pop(0) + else: + self.fail("No or invalid passport") + + if not self.premium: + m = re.search(self.WAIT_PATTERN, self.html) + if m: + wait_time = int(m.group(1)) + 1 + self.setWait(wait_time, False) + else: + wait_time = 0 + + self.captcha = self.handleCaptcha(inputs) + + if wait_time: + self.wait() + + self.errmsg = None + return inputs + + else: + inputs['referer'] = self.pyfile.url + + if self.premium: + inputs['method_premium'] = "Premium Download" + if 'method_free' in inputs: + del inputs['method_free'] + else: + inputs['method_free'] = "Free Download" + if 'method_premium' in inputs: + del inputs['method_premium'] + + self.html = self.load(self.pyfile.url, post=inputs, ref=True) + self.errmsg = None + + else: + self.parseError('FORM: %s' % (inputs['op'] if 'op' in inputs else 'UNKNOWN')) + + + def handleCaptcha(self, inputs): + m = re.search(self.CAPTCHA_URL_PATTERN, self.html) + if m: + captcha_url = m.group(1) + inputs['code'] = self.decryptCaptcha(captcha_url) + return 1 + + m = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.DOTALL) + if m: + captcha_div = m.group(1) + self.logDebug(captcha_div) + numerals = re.findall(r'(\d)', html_unescape(captcha_div)) + inputs['code'] = "".join([a[1] for a in sorted(numerals, key=lambda num: int(num[0]))]) + self.logDebug("CAPTCHA", inputs['code'], numerals) + return 2 + + recaptcha = ReCaptcha(self) + try: + captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) + except: + captcha_key = recaptcha.detect_key() + + if captcha_key: + self.logDebug("RECAPTCHA KEY: %s" % captcha_key) + inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) + return 3 + + solvemedia = SolveMedia(self) + try: + captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) + except: + captcha_key = solvemedia.detect_key() + + if captcha_key: + inputs['adcopy_challenge'], inputs['adcopy_response'] = solvemedia.challenge(captcha_key) + return 4 + + return 0 + + +getInfo = create_getInfo(XFSPHoster) -- cgit v1.2.3 From 1c4bd7a76450052b28dba609c04c5c4fcad0d014 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 8 Oct 2014 16:34:26 +0200 Subject: [XFileSharingPro] Restore --- module/plugins/internal/XFSPAccount.py | 2 +- module/plugins/internal/XFSPHoster.py | 20 +++++--------------- 2 files changed, 6 insertions(+), 16 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index 2cda54615..fdbd8829c 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -15,7 +15,7 @@ class XFSPAccount(Account): __type__ = "account" __version__ = "0.10" - __description__ = """XFileSharingPro base account plugin""" + __description__ = """XFileSharingPro account plugin""" __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), ("Walter Purcaro", "vuolter@gmail.com")] diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 2376c1b84..30cb1536f 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -21,11 +21,11 @@ class XFSPHoster(SimpleHoster): """ __name__ = "XFSPHoster" __type__ = "hoster" - __version__ = "0.37" + __version__ = "0.01" - __pattern__ = r'^unmatchable$' + __pattern__ = None - __description__ = """XFileSharingPro base hoster plugin""" + __description__ = """XFileSharingPro hoster plugin""" __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), ("stickell", "l.stickell@yahoo.it"), ("Walter Purcaro", "vuolter@gmail.com")] @@ -33,7 +33,7 @@ class XFSPHoster(SimpleHoster): HOSTER_NAME = None - FILE_URL_REPLACEMENTS = [(r'/embed-(\w{12}).*', r'/\1')] #: support embedded files + FILE_URL_REPLACEMENTS = [] COOKIES = [(HOSTER_NAME, "lang", "english")] @@ -59,14 +59,7 @@ class XFSPHoster(SimpleHoster): def setup(self): self.chunkLimit = 1 - - if self.__name__ == "XFSPHoster": - self.multiDL = True - self.__pattern__ = self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] - self.HOSTER_NAME = re.match(self.__pattern__, self.pyfile.url).group(1).lower() - self.COOKIES = [(self.HOSTER_NAME, "lang", "english")] - else: - self.resumeDownload = self.multiDL = self.premium + self.resumeDownload = self.multiDL = self.premium def prepare(self): @@ -356,6 +349,3 @@ class XFSPHoster(SimpleHoster): return 4 return 0 - - -getInfo = create_getInfo(XFSPHoster) -- cgit v1.2.3 From ae7a7e66981456e5bbe2b54006d79b6f907be7a4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 8 Oct 2014 20:18:13 +0200 Subject: Add __license__ key attribute to plugins --- module/plugins/internal/AbstractExtractor.py | 1 + module/plugins/internal/CaptchaService.py | 4 ++++ module/plugins/internal/DeadCrypter.py | 1 + module/plugins/internal/DeadHoster.py | 1 + module/plugins/internal/MultiHoster.py | 1 + module/plugins/internal/SimpleCrypter.py | 1 + module/plugins/internal/SimpleHoster.py | 1 + module/plugins/internal/UnRar.py | 1 + module/plugins/internal/UnZip.py | 1 + module/plugins/internal/XFSPAccount.py | 1 + module/plugins/internal/XFSPHoster.py | 1 + 11 files changed, 14 insertions(+) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index fefeb4ece..2b21ee357 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -17,6 +17,7 @@ class AbtractExtractor: __version__ = "0.1" __description__ = """Abtract extractor plugin""" + __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index dad537bec..187170660 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -10,6 +10,7 @@ class CaptchaService: __version__ = "0.09" __description__ = """Base captcha service plugin""" + __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] @@ -54,6 +55,7 @@ class ReCaptcha(CaptchaService): __version__ = "0.02" __description__ = """ReCaptcha captcha service plugin""" + __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] @@ -114,6 +116,7 @@ class AdsCaptcha(CaptchaService): __version__ = "0.02" __description__ = """AdsCaptcha captcha service plugin""" + __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] @@ -175,6 +178,7 @@ class SolveMedia(CaptchaService): __version__ = "0.02" __description__ = """SolveMedia captcha service plugin""" + __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 8886be0a7..760ed61b1 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -11,6 +11,7 @@ class DeadCrypter(_Crypter): __pattern__ = None __description__ = """ Crypter is no longer available """ + __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it")] diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index fff254de7..2f0a01732 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -19,6 +19,7 @@ class DeadHoster(_Hoster): __pattern__ = None __description__ = """ Hoster is no longer available """ + __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index daab1e855..38a93c6af 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -11,6 +11,7 @@ class MultiHoster(Hook): __version__ = "0.19" __description__ = """Generic MultiHoster plugin""" + __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 2da551ea9..8139d661d 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -15,6 +15,7 @@ class SimpleCrypter(Crypter): __pattern__ = None __description__ = """Simple decrypter plugin""" + __license__ = "GPLv3" __authors__ = [("stickell", "l.stickell@yahoo.it"), ("zoidberg", "zoidberg@mujmail.cz"), ("Walter Purcaro", "vuolter@gmail.com")] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3887f1579..895584af6 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -157,6 +157,7 @@ class SimpleHoster(Hoster): __pattern__ = None __description__ = """Simple hoster plugin""" + __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), ("stickell", "l.stickell@yahoo.it"), ("Walter Purcaro", "vuolter@gmail.com")] diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 11496b606..d95d55831 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -25,6 +25,7 @@ class UnRar(AbtractExtractor): __version__ = "0.18" __description__ = """Rar extractor plugin""" + __license__ = "GPLv3" __authors__ = [("RaNaN", "RaNaN@pyload.org")] diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 71f144fc7..52b4bca27 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -11,6 +11,7 @@ class UnZip(AbtractExtractor): __version__ = "0.1" __description__ = """Zip extractor plugin""" + __license__ = "GPLv3" __authors__ = [("RaNaN", "RaNaN@pyload.org")] diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index fdbd8829c..18543b9cd 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -16,6 +16,7 @@ class XFSPAccount(Account): __version__ = "0.10" __description__ = """XFileSharingPro account plugin""" + __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), ("Walter Purcaro", "vuolter@gmail.com")] diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 30cb1536f..fdaa2eb43 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -26,6 +26,7 @@ class XFSPHoster(SimpleHoster): __pattern__ = None __description__ = """XFileSharingPro hoster plugin""" + __license__ = "GPLv3" __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), ("stickell", "l.stickell@yahoo.it"), ("Walter Purcaro", "vuolter@gmail.com")] -- cgit v1.2.3 From d9f3250b1a12e50b681b2232d8153df049b1719d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 10 Oct 2014 14:17:37 +0200 Subject: [XFSPHoster] Update LINK_PATTERN --- module/plugins/internal/XFSPHoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index fdaa2eb43..228a55c82 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -21,7 +21,7 @@ class XFSPHoster(SimpleHoster): """ __name__ = "XFSPHoster" __type__ = "hoster" - __version__ = "0.01" + __version__ = "0.02" __pattern__ = None @@ -69,8 +69,8 @@ class XFSPHoster(SimpleHoster): self.fail("Missing HOSTER_NAME") if not self.LINK_PATTERN: - pattr = r'(http://([^/]*?%s|\d+\.\d+\.\d+\.\d+)(:\d+)?(/d/|(?:/files)?/\d+/\w+/)[^"\'<]+)' - self.LINK_PATTERN = pattr % self.HOSTER_NAME + pattern = r'(https?://(www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]' + self.LINK_PATTERN = pattern % self.HOSTER_NAME if isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) -- cgit v1.2.3 From 343e246e94548cf2cc713aed92212e12d28ce658 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 10 Oct 2014 16:00:34 +0200 Subject: [CaptchaService] Fix ReCaptcha KEY pattern --- module/plugins/internal/CaptchaService.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 187170660..9d8fd85f2 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -7,7 +7,7 @@ from random import random class CaptchaService: __name__ = "CaptchaService" - __version__ = "0.09" + __version__ = "0.10" __description__ = """Base captcha service plugin""" __license__ = "GPLv3" @@ -52,15 +52,15 @@ class CaptchaService: class ReCaptcha(CaptchaService): __name__ = "ReCaptcha" - __version__ = "0.02" + __version__ = "0.03" __description__ = """ReCaptcha captcha service plugin""" __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] - KEY_PATTERN = r"https?://(?:www\.)?google\.com/recaptcha/api/challenge\?k=(?P\w+?)" - KEY_AJAX_PATTERN = r"Recaptcha\.create\s*\(\s*[\"'](?P\w+)[\"']\s*," + KEY_PATTERN = r"https?://(?:www\.)?google\.com/recaptcha/api/challenge\?k=(?P\w+)" + KEY_AJAX_PATTERN = r"Recaptcha\.create\s*\(\s*[\"'](?P\w+)" def detect_key(self, html=None): -- cgit v1.2.3 From f76e5c2336718dca9da8033ba22cd83c72c7b3b3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 11 Oct 2014 15:14:28 +0200 Subject: Pattern update 1 --- module/plugins/internal/UnRar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index d95d55831..7478d68db 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -32,9 +32,9 @@ class UnRar(AbtractExtractor): CMD = "unrar" # there are some more uncovered rar formats - re_version = re.compile(r"(UNRAR 5[\.\d]+(.*?)freeware)") + re_version = re.compile(r"(UNRAR 5[\d.]+(.*?)freeware)") re_splitfile = re.compile(r"(.*)\.part(\d+)\.rar$", re.I) - re_partfiles = re.compile(r".*\.(rar|r[0-9]+)", re.I) + re_partfiles = re.compile(r".*\.(rar|r\d+)", re.I) re_filelist = re.compile(r"(.+)\s+(\d+)\s+(\d+)\s+") re_filelist5 = re.compile(r"(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)") re_wrongpwd = re.compile("(Corrupt file or wrong password|password incorrect)", re.I) -- cgit v1.2.3 From 8939f015a688a07ec7d0bd14b6a3704f6a2cb4a0 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 11 Oct 2014 15:12:40 +0200 Subject: Pattern update 3 --- module/plugins/internal/CaptchaService.py | 4 ++-- module/plugins/internal/MultiHoster.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 9d8fd85f2..db4ed41a4 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -59,8 +59,8 @@ class ReCaptcha(CaptchaService): __authors__ = [("pyLoad Team", "admin@pyload.org")] - KEY_PATTERN = r"https?://(?:www\.)?google\.com/recaptcha/api/challenge\?k=(?P\w+)" - KEY_AJAX_PATTERN = r"Recaptcha\.create\s*\(\s*[\"'](?P\w+)" + KEY_PATTERN = r'https?://(?:www\.)?google\.com/recaptcha/api/challenge\?k=(?P\w+)' + KEY_AJAX_PATTERN = r'Recaptcha\.create\s*\(\s*["\'](?P\w+)' def detect_key(self, html=None): diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 38a93c6af..4ef43bc31 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -152,9 +152,9 @@ class MultiHoster(Hook): self.logDebug("New Hosters", ", ".join(sorted(self.new_supported))) # create new regexp - regexp = r".*(%s).*" % "|".join([x.replace(".", "\\.") for x in self.new_supported]) + regexp = r'.*(%s).*' % "|".join([x.replace(".", "\\.") for x in self.new_supported]) if hasattr(klass, "__pattern__") and isinstance(klass.__pattern__, basestring) and '://' in klass.__pattern__: - regexp = r"%s|%s" % (klass.__pattern__, regexp) + regexp = r'%s|%s' % (klass.__pattern__, regexp) self.logDebug("Regexp", regexp) -- cgit v1.2.3 From 0adc51cf4c58aa3e0f1073c28b9a8781a298fe89 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 11 Oct 2014 22:59:24 +0200 Subject: [CaptchaService] Fix KEY pattern --- module/plugins/internal/CaptchaService.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index db4ed41a4..b2b549617 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -7,7 +7,7 @@ from random import random class CaptchaService: __name__ = "CaptchaService" - __version__ = "0.10" + __version__ = "0.11" __description__ = """Base captcha service plugin""" __license__ = "GPLv3" @@ -52,15 +52,15 @@ class CaptchaService: class ReCaptcha(CaptchaService): __name__ = "ReCaptcha" - __version__ = "0.03" + __version__ = "0.04" __description__ = """ReCaptcha captcha service plugin""" __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] - KEY_PATTERN = r'https?://(?:www\.)?google\.com/recaptcha/api/challenge\?k=(?P\w+)' - KEY_AJAX_PATTERN = r'Recaptcha\.create\s*\(\s*["\'](?P\w+)' + KEY_PATTERN = r'https?://(?:www\.)?google\.com/recaptcha/api/challenge\?k=(?P[\w-]+)' + KEY_AJAX_PATTERN = r'Recaptcha\.create\s*\(\s*["\'](?P[\w-]+)' def detect_key(self, html=None): @@ -175,14 +175,14 @@ class AdsCaptcha(CaptchaService): class SolveMedia(CaptchaService): __name__ = "SolveMedia" - __version__ = "0.02" + __version__ = "0.03" __description__ = """SolveMedia captcha service plugin""" __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] - KEY_PATTERN = r'http://api\.solvemedia\.com/papi/challenge\.(no)?script\?k=(?P.+?)"' + KEY_PATTERN = r'http://api\.solvemedia\.com/papi/challenge\.(no)?script\?k=(?P.+?)["\']' def challenge(self, key=None): -- cgit v1.2.3 From 818ae3ce2c5ee6f47da2f508c786c3e2ab20ad45 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 12 Oct 2014 23:30:34 +0200 Subject: [XFileSharingPro] Match option --- module/plugins/internal/XFSPHoster.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 228a55c82..40bf26cfa 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -14,11 +14,6 @@ from module.utils import html_unescape class XFSPHoster(SimpleHoster): - """ - Common base for XFileSharingPro hosters like EasybytezCom, CramitIn, FiledinoCom... - Some hosters may work straight away when added to __pattern__ - However, most of them will NOT work because they are either down or running a customized version - """ __name__ = "XFSPHoster" __type__ = "hoster" __version__ = "0.02" -- cgit v1.2.3 From 77ef3e137d2d9222d0f8965adf374e6712123a99 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 14 Oct 2014 13:07:19 +0200 Subject: [XFSPHoster] TEXT_ENCODING support --- module/plugins/internal/XFSPHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 40bf26cfa..e7348d966 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSPHoster(SimpleHoster): __name__ = "XFSPHoster" __type__ = "hoster" - __version__ = "0.02" + __version__ = "0.03" __pattern__ = None @@ -90,7 +90,7 @@ class XFSPHoster(SimpleHoster): # Due to a 0.4.9 core bug self.load would use cookies even if # cookies=False. Workaround using getURL to avoid cookies. # Can be reverted in 0.4.10 as the cookies bug has been fixed. - self.html = getURL(pyfile.url, decode=True, cookies=self.COOKIES) + self.html = getURL(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) self.file_info = self.getFileInfo() except PluginParseError: self.file_info = None -- cgit v1.2.3 From 388a2f6478d42e423f1f8442d8539983f3762f22 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 14 Oct 2014 13:38:49 +0200 Subject: Improve unit detection in size pattern --- module/plugins/internal/XFSPHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index e7348d966..f1a250f85 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -35,7 +35,7 @@ class XFSPHoster(SimpleHoster): FILE_INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' FILE_NAME_PATTERN = r'[\d\.\,]+) ?(?P\w+)?\)' + FILE_SIZE_PATTERN = r'You have requested .*\((?P[\d\.\,]+) ?(?P[\w^_]+)?\)' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' -- cgit v1.2.3 From 93106fbef7579707e2918bcd9694e4a21b37d390 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 15 Oct 2014 02:04:25 +0200 Subject: [XFSPAccount] Improve TRAFFIC_LEFT_PATTERN --- module/plugins/internal/XFSPAccount.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index 18543b9cd..9191b71a0 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -13,7 +13,7 @@ from module.utils import parseFileSize class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.10" + __version__ = "0.11" __description__ = """XFileSharingPro account plugin""" __license__ = "GPLv3" @@ -26,8 +26,12 @@ class XFSPAccount(Account): COOKIES = None #: or list of tuples [(domain, name, value)] VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(.+?)' - TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?(?P.+?)' + + TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+)\s*(?P[\w^_]+)\s*' + TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found + LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' + # PREMIUM_PATTERN = r'>Renew premium<' @@ -61,13 +65,18 @@ class XFSPAccount(Account): try: traffic = re.search(self.TRAFFIC_LEFT_PATTERN, html).groupdict() - trafficsize = traffic['S'] + traffic['U'] if 'U' in traffic else traffic['S'] - if "Unlimited" in trafficsize: + if "Unlimited" in traffic['S']: trafficleft = -1 if premium is None: premium = True else: - trafficleft = parseFileSize(trafficsize) + if 'U' in traffic: + unit = traffic['U'] + elif isinstance(self.TRAFFIC_LEFT_UNIT, basestring): + unit = self.TRAFFIC_LEFT_UNIT + else: + unit = None + trafficleft = parseFileSize(traffic['S'], unit) except: pass -- cgit v1.2.3 From 268b16c9ecc588e6e884749e3fe715431367d1c9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 15 Oct 2014 13:42:32 +0200 Subject: [XFSPAccount] Fix TRAFFIC_LEFT_PATTERN --- module/plugins/internal/XFSPAccount.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index 9191b71a0..f75422ca1 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -13,7 +13,7 @@ from module.utils import parseFileSize class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.11" + __version__ = "0.12" __description__ = """XFileSharingPro account plugin""" __license__ = "GPLv3" @@ -27,7 +27,7 @@ class XFSPAccount(Account): VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(.+?)' - TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+)\s*(?P[\w^_]+)\s*' + TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+)\s*(?:(?P[\w^_]+)\s*)?' TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' -- cgit v1.2.3 From 9a4aa10da480d9bb07592352fb22ebd2acf5dc84 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 17 Oct 2014 20:51:35 +0200 Subject: New internal plugin XFSPCrypter --- module/plugins/internal/XFSPCrypter.py | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 module/plugins/internal/XFSPCrypter.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPCrypter.py b/module/plugins/internal/XFSPCrypter.py new file mode 100644 index 000000000..4c5fff4f3 --- /dev/null +++ b/module/plugins/internal/XFSPCrypter.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleCrypter import SimpleCrypter + + +class XFSPCrypter(SimpleCrypter): + __name__ = "XFSPCrypter" + __type__ = "crypter" + __version__ = "0.01" + + __pattern__ = None + + __description__ = """XFileSharingPro decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_NAME = None + + URL_REPLACEMENTS = [(r'[?/&]+$', r''), (r'(.+/[^?]*)$', r'\1?'), (r'$', r'&per_page=10000')] + + COOKIES = [(HOSTER_NAME, "lang", "english")] + + LINK_PATTERN = None + TITLE_PATTERN = r'<[tT]itle>.*?\: (.+) folder' + + OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' + + + def prepare(self): + if not self.HOSTER_NAME: + self.fail("Missing HOSTER_NAME") + + if not self.LINK_PATTERN: + pattern = r'.+?(?:)?\s*' + self.LINK_PATTERN = pattern % self.HOSTER_NAME + + super(XFSPCrypter, self).prepare() -- cgit v1.2.3 From fac6b2007a93101f8441810118f3c8d10a3db13a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 17 Oct 2014 20:53:31 +0200 Subject: [SimpleCrypter] Fix missing LINK_PATTERN --- module/plugins/internal/SimpleCrypter.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 8139d661d..73329ed65 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -51,6 +51,8 @@ class SimpleCrypter(Crypter): return the html of the page number page_n """ + LINK_PATTERN = None + TITLE_REPLACEMENTS = [("&#?\w+;", fixup)] URL_REPLACEMENTS = [] -- cgit v1.2.3 From 6f47e10a9acf00eea4b47e5b364e75e78027809c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 18 Oct 2014 13:46:19 +0200 Subject: [XFSPCrypter] Better LINK_PATTERN --- module/plugins/internal/XFSPCrypter.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPCrypter.py b/module/plugins/internal/XFSPCrypter.py index 4c5fff4f3..372657d2d 100644 --- a/module/plugins/internal/XFSPCrypter.py +++ b/module/plugins/internal/XFSPCrypter.py @@ -6,7 +6,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter class XFSPCrypter(SimpleCrypter): __name__ = "XFSPCrypter" __type__ = "crypter" - __version__ = "0.01" + __version__ = "0.02" __pattern__ = None @@ -21,19 +21,8 @@ class XFSPCrypter(SimpleCrypter): COOKIES = [(HOSTER_NAME, "lang", "english")] - LINK_PATTERN = None + LINK_PATTERN = r'<(?:td|TD) [^>]*>\s*]*>.+?(?:)?\s*' TITLE_PATTERN = r'<[tT]itle>.*?\: (.+) folder' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' - - - def prepare(self): - if not self.HOSTER_NAME: - self.fail("Missing HOSTER_NAME") - - if not self.LINK_PATTERN: - pattern = r'.+?(?:)?\s*' - self.LINK_PATTERN = pattern % self.HOSTER_NAME - - super(XFSPCrypter, self).prepare() -- cgit v1.2.3 From 5224a71f0dc30d640754a9fa64c7c54fa3ec4fb9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 18 Oct 2014 13:48:21 +0200 Subject: [XFSPAccount] Set HOSTER_NAME (instead HOSTER_URL) as the mandatory attribute --- module/plugins/internal/XFSPAccount.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index f75422ca1..713bf827d 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -13,7 +13,7 @@ from module.utils import parseFileSize class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.12" + __version__ = "0.13" __description__ = """XFileSharingPro account plugin""" __license__ = "GPLv3" @@ -21,9 +21,19 @@ class XFSPAccount(Account): ("Walter Purcaro", "vuolter@gmail.com")] - HOSTER_URL = None + """ + Following patterns should be defined by each hoster: - COOKIES = None #: or list of tuples [(domain, name, value)] + HOSTER_URL: (optional) + example: HOSTER_URL = r'linestorage.com' + + PREMIUM_PATTERN: (optional) Checks if the account is premium + example: PREMIUM_PATTERN = r'>Renew premium<' + """ + + HOSTER_NAME = None + + COOKIES = [(HOSTER_NAME, "lang", "english")] #: or list of tuples [(domain, name, value)] VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(.+?)' @@ -32,7 +42,10 @@ class XFSPAccount(Account): LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' - # PREMIUM_PATTERN = r'>Renew premium<' + + def init(self): + if not hasattr(self, "HOSTER_URL"): + self.HOSTER_URL = "http://%s/" % self.HOSTER_NAME def loadAccountInfo(self, user, req): -- cgit v1.2.3 From d0aa0ece5e506469f631b388f6c38eaaa63b63c9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 18 Oct 2014 13:52:01 +0200 Subject: Tiny code cosmetics --- module/plugins/internal/SimpleHoster.py | 10 +++++----- module/plugins/internal/XFSPHoster.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 895584af6..c2bc4909e 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -166,18 +166,18 @@ class SimpleHoster(Hoster): """ Following patterns should be defined by each hoster: - FILE_INFO_PATTERN: Name and Size of the file + FILE_INFO_PATTERN: (optional) Name and Size of the file example: FILE_INFO_PATTERN = r'(?Pfile_name) (?Pfile_size) (?Psize_unit)' or - FILE_NAME_PATTERN: Name that will be set for the file + FILE_NAME_PATTERN: (optional) Name that will be set for the file example: FILE_NAME_PATTERN = r'(?Pfile_name)' - FILE_SIZE_PATTERN: Size that will be checked for the file + FILE_SIZE_PATTERN: (optional) Size that will be checked for the file example: FILE_SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' - OFFLINE_PATTERN: Checks if the file is yet available online + OFFLINE_PATTERN: (optional) Checks if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN: Checks if the file is temporarily offline + TEMP_OFFLINE_PATTERN: (optional) Checks if the file is temporarily offline example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index f1a250f85..abf71b033 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -330,7 +330,7 @@ class XFSPHoster(SimpleHoster): captcha_key = recaptcha.detect_key() if captcha_key: - self.logDebug("RECAPTCHA KEY: %s" % captcha_key) + self.logDebug("ReCaptcha key: %s" % captcha_key) inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) return 3 @@ -341,6 +341,7 @@ class XFSPHoster(SimpleHoster): captcha_key = solvemedia.detect_key() if captcha_key: + self.logDebug("SolveMedia key: %s" % captcha_key) inputs['adcopy_challenge'], inputs['adcopy_response'] = solvemedia.challenge(captcha_key) return 4 -- cgit v1.2.3 From e2f481deb624709764768454f7cf8222b8b45af7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 18 Oct 2014 15:23:30 +0200 Subject: [SimpleCrypter] Fix account association --- module/plugins/internal/SimpleCrypter.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 73329ed65..fe644cbda 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.13" + __version__ = "0.14" __pattern__ = None @@ -63,6 +63,15 @@ class SimpleCrypter(Crypter): LOGIN_PREMIUM = False + def init(self): + self.account = self.core.accountManager.getAccountPlugin((self.__name__ + ".py").replace("Folder.py", "")) + + if self.account and self.account.canUse(): + self.user, data = self.account.selectAccount() + self.req = self.account.getAccountRequest(self.user) + self.premium = self.account.isPremium(self.user) + + def prepare(self): if self.LOGIN_ACCOUNT and not self.account: self.fail('Required account not found!') -- cgit v1.2.3 From b9cd4eb41e9b8753bca4b5a8334d7242eab98371 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 18 Oct 2014 17:44:49 +0200 Subject: [SimpleCrypter] Fix account routine --- module/plugins/internal/SimpleCrypter.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index fe644cbda..3c1dc3244 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -10,7 +10,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.14" + __version__ = "0.15" __pattern__ = None @@ -64,12 +64,15 @@ class SimpleCrypter(Crypter): def init(self): - self.account = self.core.accountManager.getAccountPlugin((self.__name__ + ".py").replace("Folder.py", "")) + account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") + account = self.core.accountManager.getAccountPlugin(account_name) - if self.account and self.account.canUse(): - self.user, data = self.account.selectAccount() - self.req = self.account.getAccountRequest(self.user) - self.premium = self.account.isPremium(self.user) + if account and account.canUse(): + self.user, data = account.selectAccount() + self.req = account.getAccountRequest(self.user) + self.premium = account.isPremium(self.user) + + self.account = account def prepare(self): -- cgit v1.2.3 From bb1f6b3cbfc34914a1a336f8f5a2e15b57e2b5f5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Oct 2014 14:35:53 +0200 Subject: parseError -> error now calls Fail instead Exception --- module/plugins/internal/SimpleCrypter.py | 9 +++++---- module/plugins/internal/SimpleHoster.py | 29 ++++++++++------------------- module/plugins/internal/XFSPHoster.py | 17 +++++++++-------- 3 files changed, 24 insertions(+), 31 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 3c1dc3244..b45e503d5 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -3,14 +3,15 @@ import re from module.plugins.Crypter import Crypter -from module.plugins.internal.SimpleHoster import PluginParseError, replace_patterns, set_cookies +from module.plugins.Plugin import Fail +from module.plugins.internal.SimpleHoster import replace_patterns, set_cookies from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.15" + __version__ = "0.16" __pattern__ = None @@ -156,5 +157,5 @@ class SimpleCrypter(Crypter): self.package_links += self.getLinks() - def parseError(self, msg): - raise PluginParseError(msg) + def error(self, reason): + raise Fail("Parse error (%s) - crypter plugin may be out of date" % reason) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index c2bc4909e..bbd37e978 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -8,7 +8,8 @@ from urlparse import urlparse from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster -from module.utils import fixup, html_unescape, parseFileSize +from module.plugins.Plugin import Fail +reasonfrom module.utils import fixup, html_unescape, parseFileSize def replace_patterns(string, ruleslist): @@ -139,20 +140,10 @@ def timestamp(): return int(time() * 1000) -class PluginParseError(Exception): - - def __init__(self, msg): - Exception.__init__(self) - self.value = 'Parse error (%s) - plugin may be out of date' % msg - - def __str__(self): - return repr(self.value) - - class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.38" + __version__ = "0.39" __pattern__ = None @@ -249,7 +240,7 @@ class SimpleHoster(Hoster): self.tempOffline() elif status != 2: self.logDebug(self.file_info) - self.parseError('File info') + self.error('File info') if name: self.pyfile.name = name @@ -272,7 +263,7 @@ class SimpleHoster(Hoster): try: m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: - self.parseError("Free download link not found") + self.error("Free download link not found") link = m.group(1) except Exception, e: @@ -288,7 +279,7 @@ class SimpleHoster(Hoster): try: m = re.search(self.LINK_PREMIUM_PATTERN, self.html) if m is None: - self.parseError("Premium download link not found") + self.error("Premium download link not found") link = m.group(1) except Exception, e: @@ -297,10 +288,6 @@ class SimpleHoster(Hoster): self.download(link, ref=True, cookies=True, disposition=True) - def parseError(self, msg): - raise PluginParseError(msg) - - def longWait(self, wait_time=None, max_tries=3): if wait_time and isinstance(wait_time, (int, long, float)): time_str = "%dh %dm" % divmod(wait_time / 60, 60) @@ -334,3 +321,7 @@ class SimpleHoster(Hoster): if seconds: self.setWait(seconds, reconnect) super(SimpleHoster, self).wait() + + + def error(self, reason): + raise Fail("Parse error (%s) - hoster plugin may be out of date" % reason) diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index abf71b033..0b656c67c 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -9,14 +9,15 @@ from urlparse import urlparse from module.network.RequestFactory import getURL from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia -from module.plugins.internal.SimpleHoster import create_getInfo, PluginParseError, replace_patterns, set_cookies, SimpleHoster +from module.plugins.internal.SimpleHoster import create_getInfo, replace_patterns, set_cookies, SimpleHoster +from module.plugins.Plugin import Fail from module.utils import html_unescape class XFSPHoster(SimpleHoster): __name__ = "XFSPHoster" __type__ = "hoster" - __version__ = "0.03" + __version__ = "0.04" __pattern__ = None @@ -92,7 +93,7 @@ class XFSPHoster(SimpleHoster): # Can be reverted in 0.4.10 as the cookies bug has been fixed. self.html = getURL(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) self.file_info = self.getFileInfo() - except PluginParseError: + except Fail: self.file_info = None self.location = self.getDirectDownloadLink() @@ -163,7 +164,7 @@ class XFSPHoster(SimpleHoster): self.html = self.load(self.pyfile.url, post=self.getPostParameters()) m = re.search(self.LINK_PATTERN, self.html) if m is None: - self.parseError('LINK_PATTERN not found') + self.error('LINK_PATTERN not found') self.startDownload(m.group(1)) @@ -184,7 +185,7 @@ class XFSPHoster(SimpleHoster): action, inputs = self.parseHtmlForm('F1') if not inputs: - self.parseError('TEXTAREA not found') + self.error('TEXTAREA not found') self.logDebug(self.HOSTER_NAME, inputs) if inputs['st'] == 'OK': self.html = self.load(action, post=inputs) @@ -196,7 +197,7 @@ class XFSPHoster(SimpleHoster): #get easybytez.com link for uploaded file m = re.search(self.OVR_LINK_PATTERN, self.html) if m is None: - self.parseError('OVR_LINK_PATTERN not found') + self.error('OVR_LINK_PATTERN not found') self.pyfile.url = m.group(1) header = self.load(self.pyfile.url, just_header=True) if 'location' in header: # Direct link @@ -261,7 +262,7 @@ class XFSPHoster(SimpleHoster): if self.errmsg: self.retry() else: - self.parseError("Form not found") + self.error("Form not found") self.logDebug(self.HOSTER_NAME, inputs) @@ -304,7 +305,7 @@ class XFSPHoster(SimpleHoster): self.errmsg = None else: - self.parseError('FORM: %s' % (inputs['op'] if 'op' in inputs else 'UNKNOWN')) + self.error('FORM: %s' % (inputs['op'] if 'op' in inputs else 'UNKNOWN')) def handleCaptcha(self, inputs): -- cgit v1.2.3 From 4bfdf03e804fcafabab0ff08b2b6ef1daec2cf91 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Oct 2014 15:11:18 +0200 Subject: Improve error method --- module/plugins/internal/SimpleCrypter.py | 7 +++++-- module/plugins/internal/SimpleHoster.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index b45e503d5..f1def8118 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -157,5 +157,8 @@ class SimpleCrypter(Crypter): self.package_links += self.getLinks() - def error(self, reason): - raise Fail("Parse error (%s) - crypter plugin may be out of date" % reason) + def error(self, reason=None, type="parse"): + if reason: + raise Fail("%s error: %s | Plugin may be out of date" % (type.capitalize(), reason)) + else: + raise Fail("%s error | Plugin out of date" % type.capitalize()) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index bbd37e978..fe9b2d523 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -323,5 +323,8 @@ class SimpleHoster(Hoster): super(SimpleHoster, self).wait() - def error(self, reason): - raise Fail("Parse error (%s) - hoster plugin may be out of date" % reason) + def error(self, reason=None, type="parse"): + if reason: + raise Fail("%s error: %s | Plugin may be out of date" % (type.capitalize(), reason)) + else: + raise Fail("%s error | Plugin out of date" % type.capitalize()) -- cgit v1.2.3 From 1f48f481740863c7697064bde286a78e977e4a1b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Oct 2014 15:12:06 +0200 Subject: Update other plugins to support self.error --- module/plugins/internal/CaptchaService.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index b2b549617..203dda82e 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -7,7 +7,7 @@ from random import random class CaptchaService: __name__ = "CaptchaService" - __version__ = "0.11" + __version__ = "0.12" __description__ = """Base captcha service plugin""" __license__ = "GPLv3" @@ -52,7 +52,7 @@ class CaptchaService: class ReCaptcha(CaptchaService): __name__ = "ReCaptcha" - __version__ = "0.04" + __version__ = "0.05" __description__ = """ReCaptcha captcha service plugin""" __license__ = "GPLv3" @@ -99,7 +99,7 @@ class ReCaptcha(CaptchaService): challenge = re.search("challenge : '(.+?)',", js).group(1) server = re.search("server : '(.+?)',", js).group(1) except: - self.plugin.parseError("ReCaptcha challenge pattern not found") + self.plugin.error("ReCaptcha challenge pattern not found") result = self.result(server, challenge) @@ -113,7 +113,7 @@ class ReCaptcha(CaptchaService): class AdsCaptcha(CaptchaService): __name__ = "AdsCaptcha" - __version__ = "0.02" + __version__ = "0.03" __description__ = """AdsCaptcha captcha service plugin""" __license__ = "GPLv3" @@ -161,7 +161,7 @@ class AdsCaptcha(CaptchaService): challenge = re.search("challenge: '(.+?)',", js).group(1) server = re.search("server: '(.+?)',", js).group(1) except: - self.plugin.parseError("AdsCaptcha challenge pattern not found") + self.plugin.error("AdsCaptcha challenge pattern not found") result = self.result(server, challenge) @@ -175,7 +175,7 @@ class AdsCaptcha(CaptchaService): class SolveMedia(CaptchaService): __name__ = "SolveMedia" - __version__ = "0.03" + __version__ = "0.04" __description__ = """SolveMedia captcha service plugin""" __license__ = "GPLv3" @@ -200,7 +200,7 @@ class SolveMedia(CaptchaService): html).group(1) server = "http://api.solvemedia.com/papi/media" except: - self.plugin.parseError("SolveMedia challenge pattern not found") + self.plugin.error("SolveMedia challenge pattern not found") result = self.result(server, challenge) -- cgit v1.2.3 From 39b304c6495e6a77562e9ce2c9cdf921851c668a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 19 Oct 2014 16:28:10 +0200 Subject: Fixes --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index fe9b2d523..c62ef53b4 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -9,7 +9,7 @@ from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster from module.plugins.Plugin import Fail -reasonfrom module.utils import fixup, html_unescape, parseFileSize +from module.utils import fixup, html_unescape, parseFileSize def replace_patterns(string, ruleslist): @@ -143,7 +143,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.39" + __version__ = "0.40" __pattern__ = None -- cgit v1.2.3 From f70cd18b0c7cb303597a06becc791035f7fd7bd6 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 20 Oct 2014 02:20:56 +0200 Subject: [XFSPAccount] Fixed broken login due missing HOSTER_URL --- module/plugins/internal/XFSPAccount.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index 713bf827d..c2d779c29 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -13,7 +13,7 @@ from module.utils import parseFileSize class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.13" + __version__ = "0.14" __description__ = """XFileSharingPro account plugin""" __license__ = "GPLv3" @@ -43,6 +43,11 @@ class XFSPAccount(Account): LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' + def __init__(self, manager, accounts): #@TODO: remove in 0.4.10 + self.init() + return super(XFSPAccount, self).__init__(manager, accounts) + + def init(self): if not hasattr(self, "HOSTER_URL"): self.HOSTER_URL = "http://%s/" % self.HOSTER_NAME -- cgit v1.2.3 From 4e0e93fe8a9c9a98f3f299afacf07fbc855ef88b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 20 Oct 2014 12:59:27 +0200 Subject: Preload self.html --- module/plugins/internal/SimpleCrypter.py | 10 +++++----- module/plugins/internal/SimpleHoster.py | 11 ++++------- module/plugins/internal/XFSPHoster.py | 9 +++------ 3 files changed, 12 insertions(+), 18 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index f1def8118..b09ee6ac2 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -2,6 +2,7 @@ import re +from module.network.RequestFactory import getURL from module.plugins.Crypter import Crypter from module.plugins.Plugin import Fail from module.plugins.internal.SimpleHoster import replace_patterns, set_cookies @@ -11,7 +12,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.16" + __version__ = "0.17" __pattern__ = None @@ -86,14 +87,13 @@ class SimpleCrypter(Crypter): if isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) + url = self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) + self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) + def decrypt(self, pyfile): self.prepare() - pyfile.url = replace_patterns(pyfile.url, self.URL_REPLACEMENTS) - - self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING) - self.checkOnline() package_name, folder_name = self.getPackageNameAndFolder() diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index c62ef53b4..6fc61460a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -204,17 +204,16 @@ class SimpleHoster(Hoster): def prepare(self): if isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) + self.req.setOption("timeout", 120) + url = self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) + self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) + def process(self, pyfile): self.prepare() - pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS) - - # Due to a 0.4.9 core bug self.load would keep previous cookies even if overridden by cookies parameter. - # Workaround using getURL. Can be reverted in 0.4.10 as the cookies bug has been fixed. - self.html = getURL(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) if not premium_only: # Usually premium only pages doesn't show the file information self.getFileInfo() @@ -224,8 +223,6 @@ class SimpleHoster(Hoster): elif premium_only: self.fail("This link require a premium account") else: - # This line is required due to the getURL workaround. Can be removed in 0.4.10 - self.html = self.load(pyfile.url, decode=not self.TEXT_ENCODING) self.handleFree() diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 0b656c67c..14c8302e1 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -75,12 +75,13 @@ class XFSPHoster(SimpleHoster): self.errmsg = None self.passwords = self.getPassword().splitlines() + url = self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) + self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) + def process(self, pyfile): self.prepare() - pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS) - if not re.match(self.__pattern__, pyfile.url): if self.premium: self.handleOverriden() @@ -88,10 +89,6 @@ class XFSPHoster(SimpleHoster): self.fail("Only premium users can download from other hosters with %s" % self.HOSTER_NAME) else: try: - # Due to a 0.4.9 core bug self.load would use cookies even if - # cookies=False. Workaround using getURL to avoid cookies. - # Can be reverted in 0.4.10 as the cookies bug has been fixed. - self.html = getURL(pyfile.url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) self.file_info = self.getFileInfo() except Fail: self.file_info = None -- cgit v1.2.3 From 99f1eff391ca832740d1d519e8d6724a00fba952 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 02:04:46 +0200 Subject: [SimpleCrypter] Fix missing TEXT_ENCODING routine --- module/plugins/internal/SimpleCrypter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index b09ee6ac2..e2e08bb1b 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -12,7 +12,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.17" + __version__ = "0.18" __pattern__ = None @@ -88,7 +88,7 @@ class SimpleCrypter(Crypter): set_cookies(self.req.cj, self.COOKIES) url = self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) + self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) def decrypt(self, pyfile): @@ -127,6 +127,9 @@ class SimpleCrypter(Crypter): def getPackageNameAndFolder(self): + if isinstance(self.TEXT_ENCODING, basestring): + self.html = unicode(html, self.TEXT_ENCODING) + if hasattr(self, 'TITLE_PATTERN'): try: m = re.search(self.TITLE_PATTERN, self.html) -- cgit v1.2.3 From b409e7782d48830489f47cc2364b4728d98f363a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 02:06:31 +0200 Subject: Improve COOKIES parsing --- module/plugins/internal/SimpleHoster.py | 5 +++-- module/plugins/internal/XFSPHoster.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6fc61460a..74910692f 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -67,7 +67,7 @@ def parseHtmlForm(attr_str, html, input_names=None): return {}, None # no matching form found -def parseFileInfo(self, url='', html=''): +def parseFileInfo(self, url="", html=""): info = {"name": url, "size": 0, "status": 3} if hasattr(self, "pyfile"): @@ -129,6 +129,7 @@ def create_getInfo(plugin): cj = CookieJar(plugin.__name__) if isinstance(plugin.COOKIES, list): set_cookies(cj, plugin.COOKIES) + file_info = parseFileInfo(plugin, url, getURL(replace_patterns(url, plugin.FILE_URL_REPLACEMENTS), decode=not plugin.TEXT_ENCODING, cookies=cj)) yield file_info @@ -208,7 +209,7 @@ class SimpleHoster(Hoster): self.req.setOption("timeout", 120) url = self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) - self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) + self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) def process(self, pyfile): diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 14c8302e1..5d75e41a7 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -76,7 +76,7 @@ class XFSPHoster(SimpleHoster): self.passwords = self.getPassword().splitlines() url = self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) - self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=self.COOKIES) + self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) def process(self, pyfile): -- cgit v1.2.3 From 6badd4f1f210d2e9385c4d73c5b41ee5a6f04e65 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 09:59:16 +0200 Subject: Improve error method + fix TEXT_ENCODING in SimpleCrypter --- module/plugins/internal/SimpleCrypter.py | 15 +++++++++------ module/plugins/internal/SimpleHoster.py | 9 +++++---- 2 files changed, 14 insertions(+), 10 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index e2e08bb1b..2ea8d302b 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -2,6 +2,8 @@ import re +from traceback import print_exc + from module.network.RequestFactory import getURL from module.plugins.Crypter import Crypter from module.plugins.Plugin import Fail @@ -12,7 +14,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.18" + __version__ = "0.19" __pattern__ = None @@ -65,6 +67,7 @@ class SimpleCrypter(Crypter): LOGIN_PREMIUM = False + #@TODO: remove in 0.4.10 def init(self): account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") account = self.core.accountManager.getAccountPlugin(account_name) @@ -128,7 +131,7 @@ class SimpleCrypter(Crypter): def getPackageNameAndFolder(self): if isinstance(self.TEXT_ENCODING, basestring): - self.html = unicode(html, self.TEXT_ENCODING) + self.html = unicode(self.html, self.TEXT_ENCODING) if hasattr(self, 'TITLE_PATTERN'): try: @@ -160,8 +163,8 @@ class SimpleCrypter(Crypter): self.package_links += self.getLinks() + #@TODO: remove in 0.4.10 def error(self, reason=None, type="parse"): - if reason: - raise Fail("%s error: %s | Plugin may be out of date" % (type.capitalize(), reason)) - else: - raise Fail("%s error | Plugin out of date" % type.capitalize()) + raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ':' + str(reason) if reason else "")) + if self.core.debug: + print_exc() diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 74910692f..17543ba8d 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -3,6 +3,7 @@ import re from time import time +from traceback import print_exc from urlparse import urlparse from module.network.CookieJar import CookieJar @@ -321,8 +322,8 @@ class SimpleHoster(Hoster): super(SimpleHoster, self).wait() + #@TODO: remove in 0.4.10 def error(self, reason=None, type="parse"): - if reason: - raise Fail("%s error: %s | Plugin may be out of date" % (type.capitalize(), reason)) - else: - raise Fail("%s error | Plugin out of date" % type.capitalize()) + raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ':' + str(reason) if reason else "")) + if self.core.debug: + print_exc() -- cgit v1.2.3 From fe5c86946dfc2c539b4e2a59128275c1fe506493 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 14:31:45 +0200 Subject: [CaptchaService] Fix KEY_PATTERN --- module/plugins/internal/CaptchaService.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 203dda82e..28374899d 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -7,7 +7,7 @@ from random import random class CaptchaService: __name__ = "CaptchaService" - __version__ = "0.12" + __version__ = "0.13" __description__ = """Base captcha service plugin""" __license__ = "GPLv3" @@ -52,14 +52,14 @@ class CaptchaService: class ReCaptcha(CaptchaService): __name__ = "ReCaptcha" - __version__ = "0.05" + __version__ = "0.06" __description__ = """ReCaptcha captcha service plugin""" __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] - KEY_PATTERN = r'https?://(?:www\.)?google\.com/recaptcha/api/challenge\?k=(?P[\w-]+)' + KEY_PATTERN = r'recaptcha/api/challenge\?k=(?P[\w-]+)' KEY_AJAX_PATTERN = r'Recaptcha\.create\s*\(\s*["\'](?P[\w-]+)' @@ -113,15 +113,15 @@ class ReCaptcha(CaptchaService): class AdsCaptcha(CaptchaService): __name__ = "AdsCaptcha" - __version__ = "0.03" + __version__ = "0.04" __description__ = """AdsCaptcha captcha service plugin""" __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] - ID_PATTERN = r'http://api\.adscaptcha\.com/Get\.aspx\?[^"\']*CaptchaId=(?P\d+)' - KEY_PATTERN = r'http://api\.adscaptcha\.com/Get\.aspx\?[^"\']*PublicKey=(?P[\w-]+)' + ID_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*CaptchaId=(?P\d+)' + KEY_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*PublicKey=(?P[\w-]+)' def detect_key(self, html=None): @@ -175,14 +175,14 @@ class AdsCaptcha(CaptchaService): class SolveMedia(CaptchaService): __name__ = "SolveMedia" - __version__ = "0.04" + __version__ = "0.05" __description__ = """SolveMedia captcha service plugin""" __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] - KEY_PATTERN = r'http://api\.solvemedia\.com/papi/challenge\.(no)?script\?k=(?P.+?)["\']' + KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(no)?script\?k=(?P.+?)["\']' def challenge(self, key=None): -- cgit v1.2.3 From 1b096b2eb2634e8dea80b06ab9ecde206b198b35 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 19:47:00 +0200 Subject: Spare code cosmetics --- module/plugins/internal/CaptchaService.py | 22 +++++++++++----------- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 2 +- module/plugins/internal/XFSPHoster.py | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 28374899d..6d3fc0feb 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -16,7 +16,7 @@ class CaptchaService: KEY_PATTERN = None - key = None + key = None #: last key detected def __init__(self, plugin): @@ -28,7 +28,7 @@ class CaptchaService: if hasattr(self.plugin, "html") and self.plugin.html: html = self.plugin.html else: - errmsg = "%s html missing" % self.__name__ + errmsg = "%s html not found" % self.__name__ self.plugin.fail(errmsg) raise TypeError(errmsg) @@ -68,7 +68,7 @@ class ReCaptcha(CaptchaService): if hasattr(self.plugin, "html") and self.plugin.html: html = self.plugin.html else: - errmsg = "ReCaptcha html missing" + errmsg = "ReCaptcha not found" self.plugin.fail(errmsg) raise TypeError(errmsg) @@ -86,10 +86,10 @@ class ReCaptcha(CaptchaService): def challenge(self, key=None): if not key: - if self.key: + if self.detect_key(): key = self.key else: - errmsg = "ReCaptcha key missing" + errmsg = "ReCaptcha key not found" self.plugin.fail(errmsg) raise TypeError(errmsg) @@ -129,7 +129,7 @@ class AdsCaptcha(CaptchaService): if hasattr(self.plugin, "html") and self.plugin.html: html = self.plugin.html else: - errmsg = "AdsCaptcha html missing" + errmsg = "AdsCaptcha html not found" self.plugin.fail(errmsg) raise TypeError(errmsg) @@ -144,12 +144,12 @@ class AdsCaptcha(CaptchaService): return None - def challenge(self, key=None): #: key is tuple(CaptchaId, PublicKey) + def challenge(self, key=None): #: key is a tuple(CaptchaId, PublicKey) if not key: - if self.key: + if self.detect_key(): key = self.key else: - errmsg = "AdsCaptcha key missing" + errmsg = "AdsCaptcha key not found" self.plugin.fail(errmsg) raise TypeError(errmsg) @@ -187,10 +187,10 @@ class SolveMedia(CaptchaService): def challenge(self, key=None): if not key: - if self.key: + if self.detect_key(): key = self.key else: - errmsg = "SolveMedia key missing" + errmsg = "SolveMedia key not found" self.plugin.fail(errmsg) raise TypeError(errmsg) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 2ea8d302b..613ffce1f 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -165,6 +165,6 @@ class SimpleCrypter(Crypter): #@TODO: remove in 0.4.10 def error(self, reason=None, type="parse"): - raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ':' + str(reason) if reason else "")) + raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ': ' + str(reason) if reason else "")) if self.core.debug: print_exc() diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 17543ba8d..a8a44d953 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -324,6 +324,6 @@ class SimpleHoster(Hoster): #@TODO: remove in 0.4.10 def error(self, reason=None, type="parse"): - raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ':' + str(reason) if reason else "")) + raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ': ' + str(reason) if reason else "")) if self.core.debug: print_exc() diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 5d75e41a7..91a69116c 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -46,7 +46,7 @@ class XFSPHoster(SimpleHoster): OVR_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' LINK_PATTERN = None #: final download url pattern - CAPTCHA_URL_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' + CAPTCHA_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' CAPTCHA_DIV_PATTERN = r'>Enter code.*?(.+?)
' RECAPTCHA_PATTERN = None SOLVEMEDIA_PATTERN = None @@ -306,7 +306,7 @@ class XFSPHoster(SimpleHoster): def handleCaptcha(self, inputs): - m = re.search(self.CAPTCHA_URL_PATTERN, self.html) + m = re.search(self.CAPTCHA_PATTERN, self.html) if m: captcha_url = m.group(1) inputs['code'] = self.decryptCaptcha(captcha_url) -- cgit v1.2.3 From 0eb6e7ec4a1144dcca824d8add049787d3da1762 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 19:44:59 +0200 Subject: Two space before function declaration --- module/plugins/internal/AbstractExtractor.py | 8 ++++++++ module/plugins/internal/MultiHoster.py | 11 +++++++++++ module/plugins/internal/SimpleCrypter.py | 1 + module/plugins/internal/UnZip.py | 3 +++ 4 files changed, 23 insertions(+) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index 2b21ee357..8282e6ff5 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -28,6 +28,7 @@ class AbtractExtractor: """ return True + @staticmethod def getTargets(files_ids): """ Filter suited targets from list of filename id tuple list @@ -36,6 +37,7 @@ class AbtractExtractor: """ raise NotImplementedError + def __init__(self, m, file, out, fullpath, overwrite, excludefiles, renice): """Initialize extractor for specific file @@ -55,10 +57,12 @@ class AbtractExtractor: self.renice = renice self.files = [] #: Store extracted files here + def init(self): """ Initialize additional data structures """ pass + def checkArchive(self): """Check if password if needed. Raise ArchiveError if integrity is questionable. @@ -68,6 +72,7 @@ class AbtractExtractor: """ return False + def checkPassword(self, password): """ Check if the given password is/might be correct. If it can not be decided at this point return true. @@ -77,6 +82,7 @@ class AbtractExtractor: """ return True + def extract(self, progress, password=None): """Extract the archive. Raise specific errors in case of failure. @@ -89,6 +95,7 @@ class AbtractExtractor: """ raise NotImplementedError + def getDeleteFiles(self): """Return list of files to delete, do *not* delete them here. @@ -96,6 +103,7 @@ class AbtractExtractor: """ raise NotImplementedError + def getExtractedFiles(self): """Populate self.files at some point while extracting""" return self.files diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 4ef43bc31..5aadf6f2e 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -28,6 +28,7 @@ class MultiHoster(Hook): self.supported = [] self.new_supported = [] + def getConfig(self, option, default=''): """getConfig with default value - sublass may not implements all config options""" try: @@ -35,6 +36,7 @@ class MultiHoster(Hook): except KeyError: return default + def getHosterCached(self): if not self.hosters: @@ -61,6 +63,7 @@ class MultiHoster(Hook): return self.hosters + def toHosterSet(self, hosters): hosters = set((str(x).strip().lower() for x in hosters)) @@ -72,6 +75,7 @@ class MultiHoster(Hook): hosters.discard('') return hosters + def getHoster(self): """Load list of supported hoster @@ -79,6 +83,7 @@ class MultiHoster(Hook): """ raise NotImplementedError + def coreReady(self): if self.cb: self.core.scheduler.removeJob(self.cb) @@ -94,9 +99,11 @@ class MultiHoster(Hook): else: self.periodical() + def initPeriodical(self): pass + def periodical(self): """reload hoster list periodically""" self.logInfo(_("Reloading supported hoster list")) @@ -112,6 +119,7 @@ class MultiHoster(Hook): for hoster in old_supported: self.unloadHoster(hoster) + def overridePlugins(self): pluginMap = {} for name in self.core.pluginManager.hosterPlugins.keys(): @@ -162,6 +170,7 @@ class MultiHoster(Hook): dict['pattern'] = regexp dict['re'] = re.compile(regexp) + def unloadHoster(self, hoster): dict = self.core.pluginManager.hosterPlugins[hoster] if "module" in dict: @@ -171,6 +180,7 @@ class MultiHoster(Hook): del dict['new_module'] del dict['new_name'] + def unload(self): """Remove override for all hosters. Scheduler job is removed by hookmanager""" for hoster in self.supported: @@ -182,6 +192,7 @@ class MultiHoster(Hook): dict['pattern'] = getattr(klass, "__pattern__", r'^unmatchable$') dict['re'] = re.compile(dict['pattern']) + def downloadFailed(self, pyfile): """remove plugin override if download fails but not if file is offline/temp.offline""" if pyfile.hasStatus("failed") and self.getConfig("unloadFailing", True): diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 613ffce1f..6c00a2267 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -51,6 +51,7 @@ class SimpleCrypter(Crypter): and its loadPage method: + def loadPage(self, page_n): return the html of the page number page_n """ diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 52b4bca27..ec493bcd8 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -19,6 +19,7 @@ class UnZip(AbtractExtractor): def checkDeps(): return sys.version_info[:2] >= (2, 6) + @staticmethod def getTargets(files_ids): result = [] @@ -29,10 +30,12 @@ class UnZip(AbtractExtractor): return result + def extract(self, progress, password=None): z = zipfile.ZipFile(self.file) self.files = z.namelist() z.extractall(self.out) + def getDeleteFiles(self): return [self.file] -- cgit v1.2.3 From 85ba09f7bd27461ab5900487fd9da705cc7ff3a4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 21:14:50 +0200 Subject: [CaptchaService] Update ReCaptcha KEY_PATTERN --- module/plugins/internal/CaptchaService.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 6d3fc0feb..11b34755b 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -7,7 +7,7 @@ from random import random class CaptchaService: __name__ = "CaptchaService" - __version__ = "0.13" + __version__ = "0.14" __description__ = """Base captcha service plugin""" __license__ = "GPLv3" @@ -52,14 +52,14 @@ class CaptchaService: class ReCaptcha(CaptchaService): __name__ = "ReCaptcha" - __version__ = "0.06" + __version__ = "0.07" __description__ = """ReCaptcha captcha service plugin""" __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] - KEY_PATTERN = r'recaptcha/api/challenge\?k=(?P[\w-]+)' + KEY_PATTERN = r'recaptcha(/api|\.net)/(challenge|noscript)\?k=(?P[\w-]+)' KEY_AJAX_PATTERN = r'Recaptcha\.create\s*\(\s*["\'](?P[\w-]+)' -- cgit v1.2.3 From e41f3d7f9bac5c5923d5e3e4ae4b373f6c90dc16 Mon Sep 17 00:00:00 2001 From: guidobelix Date: Wed, 22 Oct 2014 21:41:11 +0200 Subject: Broken TRAFFIC_LEFT_PATTERN in RapidfileshareNet and TusfilesNet account --- module/plugins/internal/XFSPAccount.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index c2d779c29..d420adbb6 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -98,10 +98,7 @@ class XFSPAccount(Account): except: pass - if premium is None: - premium = False - - return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} + return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium or False} def login(self, user, data, req): -- cgit v1.2.3 From 822fc8edc732248e5e79bf5a0ef9555bcc56b21c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 23:08:05 +0200 Subject: [XFSPAccount] Better HOSTER_URL --- module/plugins/internal/XFSPAccount.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index d420adbb6..5e2a0afc4 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -13,7 +13,7 @@ from module.utils import parseFileSize class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.14" + __version__ = "0.15" __description__ = """XFileSharingPro account plugin""" __license__ = "GPLv3" @@ -50,7 +50,7 @@ class XFSPAccount(Account): def init(self): if not hasattr(self, "HOSTER_URL"): - self.HOSTER_URL = "http://%s/" % self.HOSTER_NAME + self.HOSTER_URL = "http://www.%s/" % self.HOSTER_NAME.replace("www.", "", 1) def loadAccountInfo(self, user, req): -- cgit v1.2.3 From 7b67f68c1636014fe02286da71bdc6c13dc3eeee Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 23 Oct 2014 00:50:23 +0200 Subject: Simplify captcha challenge calls --- module/plugins/internal/CaptchaService.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 11b34755b..a1ec633e8 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -29,7 +29,7 @@ class CaptchaService: html = self.plugin.html else: errmsg = "%s html not found" % self.__name__ - self.plugin.fail(errmsg) + self.plugin.fail(errmsg) #@TODO: replace all plugin.fail(errmsg) with plugin.error(errmsg) in 0.4.10 raise TypeError(errmsg) m = re.search(self.KEY_PATTERN, html) @@ -72,9 +72,7 @@ class ReCaptcha(CaptchaService): self.plugin.fail(errmsg) raise TypeError(errmsg) - m = re.search(self.KEY_PATTERN, html) - if m is None: - m = re.search(self.KEY_AJAX_PATTERN, html) + m = re.search(self.KEY_PATTERN, html) or re.search(self.KEY_AJAX_PATTERN, html) if m: self.key = m.group("KEY") self.plugin.logDebug("ReCaptcha key: %s" % self.key) -- cgit v1.2.3 From 8e89cb7781ec3b135feeb750f6321bbc85e89d67 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 23 Oct 2014 22:54:40 +0200 Subject: Fix error method --- module/plugins/internal/SimpleCrypter.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 6c00a2267..402a641ea 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -165,7 +165,7 @@ class SimpleCrypter(Crypter): #@TODO: remove in 0.4.10 - def error(self, reason=None, type="parse"): - raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ': ' + str(reason) if reason else "")) + def error(self, reason="", type="parse"): if self.core.debug: print_exc() + raise Fail("%s error%s | Plugin may be out of date" % (type.strip().capitalize(), ': ' + reason.strip() if reason else "")) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index a8a44d953..be3e3c8fc 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -323,7 +323,7 @@ class SimpleHoster(Hoster): #@TODO: remove in 0.4.10 - def error(self, reason=None, type="parse"): - raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ': ' + str(reason) if reason else "")) + def error(self, reason="", type="parse"): if self.core.debug: print_exc() + raise Fail("%s error%s | Plugin may be out of date" % (type.strip().capitalize(), ': ' + reason.strip() if reason else "")) -- cgit v1.2.3 From 29de13dc5098cc9214f3f8f111da140a22e3e2ff Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Oct 2014 01:15:58 +0200 Subject: Use parseTraffic instead parseFileSize in accounts --- module/plugins/internal/XFSPAccount.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index 5e2a0afc4..e54fda987 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -7,13 +7,12 @@ from time import mktime, strptime from module.plugins.Account import Account from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies -from module.utils import parseFileSize class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.15" + __version__ = "0.16" __description__ = """XFileSharingPro account plugin""" __license__ = "GPLv3" @@ -94,7 +93,7 @@ class XFSPAccount(Account): unit = self.TRAFFIC_LEFT_UNIT else: unit = None - trafficleft = parseFileSize(traffic['S'], unit) + trafficleft = self.parseTraffic(traffic['S'], unit) except: pass -- cgit v1.2.3 From c41d4ec4c5935ee702a44e919730971028bf5724 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Oct 2014 01:16:52 +0200 Subject: Fix and improve plugins logging --- module/plugins/internal/SimpleHoster.py | 2 ++ module/plugins/internal/XFSPHoster.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index be3e3c8fc..51010d569 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -221,10 +221,12 @@ class SimpleHoster(Hoster): self.getFileInfo() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handle as premium download") self.handlePremium() elif premium_only: self.fail("This link require a premium account") else: + self.logDebug("Handle as free download") self.handleFree() diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 91a69116c..87a82beaf 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -102,8 +102,10 @@ class XFSPHoster(SimpleHoster): if self.location: self.startDownload(self.location) elif self.premium: + self.logDebug("Handle as premium download") self.handlePremium() else: + self.logDebug("Handle as free download") self.handleFree() @@ -126,7 +128,6 @@ class XFSPHoster(SimpleHoster): def handleFree(self): url = self.getDownloadLink() - self.logDebug("Download URL: %s" % url) self.startDownload(url) @@ -312,7 +313,7 @@ class XFSPHoster(SimpleHoster): inputs['code'] = self.decryptCaptcha(captcha_url) return 1 - m = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.DOTALL) + m = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.S) if m: captcha_div = m.group(1) self.logDebug(captcha_div) -- cgit v1.2.3 From 4da90891eb2544ac15a7d512aba8cb357f68ee5f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Oct 2014 01:11:29 +0200 Subject: Spare code cosmetics --- module/plugins/internal/UnRar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 7478d68db..d8bb6f251 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -168,9 +168,9 @@ class UnRar(AbtractExtractor): def getDeleteFiles(self): if ".part" in basename(self.file): - return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.IGNORECASE)) + return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.I)) # get files which matches .r* and filter unsuited files out - parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.IGNORECASE)) + parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.I)) return filter(lambda x: self.re_partfiles.match(x), parts) -- cgit v1.2.3 From e3f5280529921100f48bb8a79853bf480c7611e4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Oct 2014 02:53:05 +0200 Subject: Replace single quotes with doubles in self.error and self.fail msg --- module/plugins/internal/SimpleCrypter.py | 6 +++--- module/plugins/internal/SimpleHoster.py | 2 +- module/plugins/internal/XFSPHoster.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 402a641ea..3cbb0f70a 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -83,10 +83,10 @@ class SimpleCrypter(Crypter): def prepare(self): if self.LOGIN_ACCOUNT and not self.account: - self.fail('Required account not found!') + self.fail("Required account not found!") if self.LOGIN_PREMIUM and not self.premium: - self.fail('Required premium account not found!') + self.fail("Required premium account not found!") if isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) @@ -112,7 +112,7 @@ class SimpleCrypter(Crypter): if self.package_links: self.packages = [(package_name, self.package_links, folder_name)] else: - self.fail('Could not extract any links') + self.fail("Could not extract any links") def getLinks(self): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 51010d569..951a05783 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -241,7 +241,7 @@ class SimpleHoster(Hoster): self.tempOffline() elif status != 2: self.logDebug(self.file_info) - self.error('File info') + self.error("File info") if name: self.pyfile.name = name diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 87a82beaf..fd39bed02 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -162,7 +162,7 @@ class XFSPHoster(SimpleHoster): self.html = self.load(self.pyfile.url, post=self.getPostParameters()) m = re.search(self.LINK_PATTERN, self.html) if m is None: - self.error('LINK_PATTERN not found') + self.error("LINK_PATTERN not found") self.startDownload(m.group(1)) @@ -183,7 +183,7 @@ class XFSPHoster(SimpleHoster): action, inputs = self.parseHtmlForm('F1') if not inputs: - self.error('TEXTAREA not found') + self.error("TEXTAREA not found") self.logDebug(self.HOSTER_NAME, inputs) if inputs['st'] == 'OK': self.html = self.load(action, post=inputs) @@ -195,7 +195,7 @@ class XFSPHoster(SimpleHoster): #get easybytez.com link for uploaded file m = re.search(self.OVR_LINK_PATTERN, self.html) if m is None: - self.error('OVR_LINK_PATTERN not found') + self.error("OVR_LINK_PATTERN not found") self.pyfile.url = m.group(1) header = self.load(self.pyfile.url, just_header=True) if 'location' in header: # Direct link @@ -303,7 +303,7 @@ class XFSPHoster(SimpleHoster): self.errmsg = None else: - self.error('FORM: %s' % (inputs['op'] if 'op' in inputs else 'UNKNOWN')) + self.error("FORM: %s" % (inputs['op'] if 'op' in inputs else 'UNKNOWN')) def handleCaptcha(self, inputs): -- cgit v1.2.3 From 1c4bf83881d2a22da3773666a580d51f6b57bfd1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Oct 2014 03:07:21 +0200 Subject: Avoid gettext conflict due variable `_` --- module/plugins/internal/XFSPHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index fd39bed02..19cddba64 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -245,7 +245,7 @@ class XFSPHoster(SimpleHoster): def getPostParameters(self): - for _ in xrange(3): + for _i in xrange(3): if not self.errmsg: self.checkErrors() -- cgit v1.2.3 From 9f2ebe486a3e155fb6a60e07cccb77ab6a772eb2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 26 Oct 2014 02:31:54 +0200 Subject: Extend translation support in plugins + a lot of code cosmetics and typo fixes --- module/plugins/internal/CaptchaService.py | 12 ++++++------ module/plugins/internal/SimpleCrypter.py | 6 +++--- module/plugins/internal/SimpleHoster.py | 12 ++++++------ module/plugins/internal/XFSPHoster.py | 26 +++++++++++++------------- 4 files changed, 28 insertions(+), 28 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index a1ec633e8..aa8758344 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -28,7 +28,7 @@ class CaptchaService: if hasattr(self.plugin, "html") and self.plugin.html: html = self.plugin.html else: - errmsg = "%s html not found" % self.__name__ + errmsg = _("%s html not found") % self.__name__ self.plugin.fail(errmsg) #@TODO: replace all plugin.fail(errmsg) with plugin.error(errmsg) in 0.4.10 raise TypeError(errmsg) @@ -68,7 +68,7 @@ class ReCaptcha(CaptchaService): if hasattr(self.plugin, "html") and self.plugin.html: html = self.plugin.html else: - errmsg = "ReCaptcha not found" + errmsg = _("ReCaptcha html not found") self.plugin.fail(errmsg) raise TypeError(errmsg) @@ -87,7 +87,7 @@ class ReCaptcha(CaptchaService): if self.detect_key(): key = self.key else: - errmsg = "ReCaptcha key not found" + errmsg = _("ReCaptcha key not found") self.plugin.fail(errmsg) raise TypeError(errmsg) @@ -127,7 +127,7 @@ class AdsCaptcha(CaptchaService): if hasattr(self.plugin, "html") and self.plugin.html: html = self.plugin.html else: - errmsg = "AdsCaptcha html not found" + errmsg = _("AdsCaptcha html not found") self.plugin.fail(errmsg) raise TypeError(errmsg) @@ -147,7 +147,7 @@ class AdsCaptcha(CaptchaService): if self.detect_key(): key = self.key else: - errmsg = "AdsCaptcha key not found" + errmsg = _("AdsCaptcha key not found") self.plugin.fail(errmsg) raise TypeError(errmsg) @@ -188,7 +188,7 @@ class SolveMedia(CaptchaService): if self.detect_key(): key = self.key else: - errmsg = "SolveMedia key not found" + errmsg = _("SolveMedia key not found") self.plugin.fail(errmsg) raise TypeError(errmsg) diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 3cbb0f70a..61286449d 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -83,10 +83,10 @@ class SimpleCrypter(Crypter): def prepare(self): if self.LOGIN_ACCOUNT and not self.account: - self.fail("Required account not found!") + self.fail(_("Required account not found!")) if self.LOGIN_PREMIUM and not self.premium: - self.fail("Required premium account not found!") + self.fail(_("Required premium account not found!")) if isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) @@ -112,7 +112,7 @@ class SimpleCrypter(Crypter): if self.package_links: self.packages = [(package_name, self.package_links, folder_name)] else: - self.fail("Could not extract any links") + self.fail(_("Could not extract any links")) def getLinks(self): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 951a05783..135d3c9d9 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -224,7 +224,7 @@ class SimpleHoster(Hoster): self.logDebug("Handle as premium download") self.handlePremium() elif premium_only: - self.fail("This link require a premium account") + self.fail(_("This link require a premium account")) else: self.logDebug("Handle as free download") self.handleFree() @@ -241,7 +241,7 @@ class SimpleHoster(Hoster): self.tempOffline() elif status != 2: self.logDebug(self.file_info) - self.error("File info") + self.error(_("File info")) if name: self.pyfile.name = name @@ -259,12 +259,12 @@ class SimpleHoster(Hoster): def handleFree(self): if not hasattr(self, 'LINK_FREE_PATTERN'): - self.fail("Free download not implemented") + self.fail(_("Free download not implemented")) try: m = re.search(self.LINK_FREE_PATTERN, self.html) if m is None: - self.error("Free download link not found") + self.error(_("Free download link not found")) link = m.group(1) except Exception, e: @@ -275,12 +275,12 @@ class SimpleHoster(Hoster): def handlePremium(self): if not hasattr(self, 'LINK_PREMIUM_PATTERN'): - self.fail("Premium download not implemented") + self.fail(_("Premium download not implemented")) try: m = re.search(self.LINK_PREMIUM_PATTERN, self.html) if m is None: - self.error("Premium download link not found") + self.error(_("Premium download link not found")) link = m.group(1) except Exception, e: diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 19cddba64..0ea5a95c5 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -62,7 +62,7 @@ class XFSPHoster(SimpleHoster): def prepare(self): """ Initialize important variables """ if not self.HOSTER_NAME: - self.fail("Missing HOSTER_NAME") + self.fail(_("Missing HOSTER_NAME")) if not self.LINK_PATTERN: pattern = r'(https?://(www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]' @@ -86,7 +86,7 @@ class XFSPHoster(SimpleHoster): if self.premium: self.handleOverriden() else: - self.fail("Only premium users can download from other hosters with %s" % self.HOSTER_NAME) + self.fail(_("Only premium users can download from other hosters with %s") % self.HOSTER_NAME) else: try: self.file_info = self.getFileInfo() @@ -151,9 +151,9 @@ class XFSPHoster(SimpleHoster): else: if self.errmsg and 'captcha' in self.errmsg: - self.fail("No valid captcha code entered") + self.fail(_("No valid captcha code entered")) else: - self.fail("Download link not found") + self.fail(_("Download link not found")) return m.group(1) @@ -162,7 +162,7 @@ class XFSPHoster(SimpleHoster): self.html = self.load(self.pyfile.url, post=self.getPostParameters()) m = re.search(self.LINK_PATTERN, self.html) if m is None: - self.error("LINK_PATTERN not found") + self.error(_("LINK_PATTERN not found")) self.startDownload(m.group(1)) @@ -183,19 +183,19 @@ class XFSPHoster(SimpleHoster): action, inputs = self.parseHtmlForm('F1') if not inputs: - self.error("TEXTAREA not found") + self.error(_("TEXTAREA not found")) self.logDebug(self.HOSTER_NAME, inputs) if inputs['st'] == 'OK': self.html = self.load(action, post=inputs) elif inputs['st'] == 'Can not leech file': self.retry(max_tries=20, wait_time=3 * 60, reason=inputs['st']) else: - self.fail(inputs['st']) + self.fail(_(inputs['st'])) #get easybytez.com link for uploaded file m = re.search(self.OVR_LINK_PATTERN, self.html) if m is None: - self.error("OVR_LINK_PATTERN not found") + self.error(_("OVR_LINK_PATTERN not found")) self.pyfile.url = m.group(1) header = self.load(self.pyfile.url, just_header=True) if 'location' in header: # Direct link @@ -225,7 +225,7 @@ class XFSPHoster(SimpleHoster): elif 'captcha' in self.errmsg: self.invalidCaptcha() elif 'premium' in self.errmsg and 'require' in self.errmsg: - self.fail("File can be downloaded by premium users only") + self.fail(_("File can be downloaded by premium users only")) elif 'limit' in self.errmsg: self.wait(1 * 60 * 60, True) self.retry(25) @@ -234,7 +234,7 @@ class XFSPHoster(SimpleHoster): elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: self.tempOffline() elif 'download files up to' in self.errmsg: - self.fail("File too large for free download") + self.fail(_("File too large for free download")) else: self.fail(self.errmsg) @@ -260,7 +260,7 @@ class XFSPHoster(SimpleHoster): if self.errmsg: self.retry() else: - self.error("Form not found") + self.error(_("Form not found")) self.logDebug(self.HOSTER_NAME, inputs) @@ -269,7 +269,7 @@ class XFSPHoster(SimpleHoster): if self.passwords: inputs['password'] = self.passwords.pop(0) else: - self.fail("No or invalid passport") + self.fail(_("No or invalid passport")) if not self.premium: m = re.search(self.WAIT_PATTERN, self.html) @@ -303,7 +303,7 @@ class XFSPHoster(SimpleHoster): self.errmsg = None else: - self.error("FORM: %s" % (inputs['op'] if 'op' in inputs else 'UNKNOWN')) + self.error(_("FORM: %s") % (inputs['op'] if 'op' in inputs else _("UNKNOWN"))) def handleCaptcha(self, inputs): -- cgit v1.2.3 From 8e1663b2c6c0367e9b9afa08412ad896faa996b5 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 26 Oct 2014 02:44:28 +0100 Subject: [XFSPHoster] Fix checkErrors --- module/plugins/internal/XFSPHoster.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 0ea5a95c5..00666ec2b 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -17,7 +17,7 @@ from module.utils import html_unescape class XFSPHoster(SimpleHoster): __name__ = "XFSPHoster" __type__ = "hoster" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = None @@ -188,7 +188,7 @@ class XFSPHoster(SimpleHoster): if inputs['st'] == 'OK': self.html = self.load(action, post=inputs) elif inputs['st'] == 'Can not leech file': - self.retry(max_tries=20, wait_time=3 * 60, reason=inputs['st']) + self.retry(20, 3 * 60, inputs['st']) else: self.fail(_(inputs['st'])) @@ -196,12 +196,14 @@ class XFSPHoster(SimpleHoster): m = re.search(self.OVR_LINK_PATTERN, self.html) if m is None: self.error(_("OVR_LINK_PATTERN not found")) + self.pyfile.url = m.group(1) + header = self.load(self.pyfile.url, just_header=True) if 'location' in header: # Direct link self.startDownload(self.pyfile.url) else: - self.retry() + self.retry(reason="OVR link location not found") def startDownload(self, link): @@ -227,10 +229,10 @@ class XFSPHoster(SimpleHoster): elif 'premium' in self.errmsg and 'require' in self.errmsg: self.fail(_("File can be downloaded by premium users only")) elif 'limit' in self.errmsg: - self.wait(1 * 60 * 60, True) - self.retry(25) + self.wantReconnect = True + self.retry(25, 1 * 60 * 60, "Download limit exceeded") elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: - self.retry() + self.retry(reason="Link expired") elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: self.tempOffline() elif 'download files up to' in self.errmsg: @@ -246,8 +248,7 @@ class XFSPHoster(SimpleHoster): def getPostParameters(self): for _i in xrange(3): - if not self.errmsg: - self.checkErrors() + self.checkErrors() if hasattr(self, "FORM_PATTERN"): action, inputs = self.parseHtmlForm(self.FORM_PATTERN) @@ -258,7 +259,7 @@ class XFSPHoster(SimpleHoster): action, inputs = self.parseHtmlForm('F1') if not inputs: if self.errmsg: - self.retry() + self.retry(reason=self.errmsg) else: self.error(_("Form not found")) -- cgit v1.2.3 From e92919794e2c7e49e015fee8dfc906404fa0e541 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 26 Oct 2014 19:35:43 +0100 Subject: [XFSPAccount] Improve VALID_UNTIL_PATTERN --- module/plugins/internal/XFSPAccount.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index e54fda987..69b5cfc22 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.16" + __version__ = "0.17" __description__ = """XFileSharingPro account plugin""" __license__ = "GPLv3" @@ -34,7 +34,7 @@ class XFSPAccount(Account): COOKIES = [(HOSTER_NAME, "lang", "english")] #: or list of tuples [(domain, name, value)] - VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(.+?)' + VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?([^><]+)' TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+)\s*(?:(?P[\w^_]+)\s*)?' TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found -- cgit v1.2.3 From 8f0303277fce7471539508f7a8fad033e5e87076 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 26 Oct 2014 19:36:46 +0100 Subject: [SimpleHoster][SimpleCrypter] Clean error method --- module/plugins/internal/SimpleCrypter.py | 7 ++----- module/plugins/internal/SimpleHoster.py | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 9 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 61286449d..9e1b11f7b 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -7,7 +7,7 @@ from traceback import print_exc from module.network.RequestFactory import getURL from module.plugins.Crypter import Crypter from module.plugins.Plugin import Fail -from module.plugins.internal.SimpleHoster import replace_patterns, set_cookies +from module.plugins.internal.SimpleHoster import _error, replace_patterns, set_cookies from module.utils import fixup, html_unescape @@ -164,8 +164,5 @@ class SimpleCrypter(Crypter): self.package_links += self.getLinks() - #@TODO: remove in 0.4.10 def error(self, reason="", type="parse"): - if self.core.debug: - print_exc() - raise Fail("%s error%s | Plugin may be out of date" % (type.strip().capitalize(), ': ' + reason.strip() if reason else "")) + return _error(reason, type) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 135d3c9d9..84a77a312 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -13,6 +13,20 @@ from module.plugins.Plugin import Fail from module.utils import fixup, html_unescape, parseFileSize +#@TODO: Remove in 0.4.10 and redirect to self.error instead +def _error(self, reason="", type=""): + if not reason and not type: + type = "unknown" + + msg = _("%s error") % type.strip().capitalize() if type else _("Error") + msg += ": " + reason.strip() if reason else "" + msg += _(" | Plugin may be out of date") + + if self.core.debug: + print_exc() + raise Fail(msg) + + def replace_patterns(string, ruleslist): for r in ruleslist: rf, rt = r @@ -324,8 +338,5 @@ class SimpleHoster(Hoster): super(SimpleHoster, self).wait() - #@TODO: remove in 0.4.10 def error(self, reason="", type="parse"): - if self.core.debug: - print_exc() - raise Fail("%s error%s | Plugin may be out of date" % (type.strip().capitalize(), ': ' + reason.strip() if reason else "")) + return _error(reason, type) -- cgit v1.2.3 From f6aed08062c3b6a4dad09d238a32ef5ab58479a4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 26 Oct 2014 20:49:31 +0100 Subject: [SimpleHoster][SimpleCrypter] Bump up __version__ --- module/plugins/internal/SimpleCrypter.py | 2 +- module/plugins/internal/SimpleHoster.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 9e1b11f7b..31102b6fa 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -14,7 +14,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.19" + __version__ = "0.20" __pattern__ = None diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 84a77a312..87ebfc533 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -159,7 +159,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.40" + __version__ = "0.41" __pattern__ = None -- cgit v1.2.3 From d99e7039744c67ddbe56a09eeaa636f7d62e98a8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 26 Oct 2014 21:10:38 +0100 Subject: [XFSPHoster] Improve ERROR_PATTERN --- module/plugins/internal/XFSPHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 00666ec2b..edbe8a0a2 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -17,7 +17,7 @@ from module.utils import html_unescape class XFSPHoster(SimpleHoster): __name__ = "XFSPHoster" __type__ = "hoster" - __version__ = "0.05" + __version__ = "0.06" __pattern__ = None @@ -51,7 +51,7 @@ class XFSPHoster(SimpleHoster): RECAPTCHA_PATTERN = None SOLVEMEDIA_PATTERN = None - ERROR_PATTERN = r'class=["\']err["\'][^>]*>(.+?)]*>|<[Cc]enter>)(.+?)(?:["\']| Date: Mon, 27 Oct 2014 01:16:51 +0100 Subject: [SimpleHoster] Direct download link support --- module/plugins/internal/SimpleHoster.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 87ebfc533..ca84dc33e 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -159,7 +159,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.41" + __version__ = "0.42" __pattern__ = None @@ -208,6 +208,8 @@ class SimpleHoster(Hoster): COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + CHECK_DIRECT_DOWNLOAD = True #: Search for a direct download link in the page header (premium only) + def init(self): self.file_info = {} @@ -223,15 +225,22 @@ class SimpleHoster(Hoster): self.req.setOption("timeout", 120) - url = self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) - self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) + self.html = getURL(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) def process(self, pyfile): self.prepare() + if self.CHECK_DIRECT_DOWNLOAD and self.premium: + header = self.load(self.pyfile.url, just_header=True) + if 'location' in header: + self.logDebug("Direct download link detected") + self.download(header['location'], ref=True, cookies=True, disposition=True) + return + premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only: # Usually premium only pages doesn't show the file information + if not premium_only: #: Usually premium only pages doesn't show the file information self.getFileInfo() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): @@ -282,7 +291,7 @@ class SimpleHoster(Hoster): link = m.group(1) except Exception, e: - self.logError(str(e)) + self.fail(str(e)) else: self.download(link, ref=True, cookies=True, disposition=True) @@ -298,7 +307,7 @@ class SimpleHoster(Hoster): link = m.group(1) except Exception, e: - self.logError(str(e)) + self.fail(str(e)) else: self.download(link, ref=True, cookies=True, disposition=True) @@ -308,14 +317,14 @@ class SimpleHoster(Hoster): time_str = "%dh %dm" % divmod(wait_time / 60, 60) else: wait_time = 900 - time_str = "(unknown time)" + time_str = _("(unknown time)") max_tries = 100 self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str) self.setWait(wait_time, True) self.wait() - self.retry(max_tries=max_tries, reason="Download limit reached") + self.retry(max_tries=max_tries, reason=_("Download limit reached")) def parseHtmlForm(self, attr_str='', input_names=None): -- cgit v1.2.3 From 146fe1e309c33ab149bfaf58ad86c0dd4fb9b156 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Oct 2014 01:18:45 +0100 Subject: Spare code cosmetics --- module/plugins/internal/SimpleCrypter.py | 4 ++-- module/plugins/internal/XFSPHoster.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 31102b6fa..f6783cffb 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -91,8 +91,8 @@ class SimpleCrypter(Crypter): if isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) - url = self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) + self.html = getURL(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) def decrypt(self, pyfile): diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index edbe8a0a2..44b2006ae 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -201,9 +201,9 @@ class XFSPHoster(SimpleHoster): header = self.load(self.pyfile.url, just_header=True) if 'location' in header: # Direct link - self.startDownload(self.pyfile.url) + self.startDownload(header['location']) else: - self.retry(reason="OVR link location not found") + self.retry(reason=_("OVR link location not found")) def startDownload(self, link): @@ -232,7 +232,7 @@ class XFSPHoster(SimpleHoster): self.wantReconnect = True self.retry(25, 1 * 60 * 60, "Download limit exceeded") elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: - self.retry(reason="Link expired") + self.retry(reason=_("Link expired")) elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: self.tempOffline() elif 'download files up to' in self.errmsg: -- cgit v1.2.3 From aa0751bcfd995e308bcd586a6965c75e68b1274b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Oct 2014 23:05:13 +0100 Subject: Code cosmetics --- module/plugins/internal/DeadCrypter.py | 9 +++++---- module/plugins/internal/DeadHoster.py | 9 +++++---- module/plugins/internal/MultiHoster.py | 9 ++++++--- module/plugins/internal/SimpleCrypter.py | 15 +++++++-------- module/plugins/internal/SimpleHoster.py | 13 +++++++------ module/plugins/internal/XFSPAccount.py | 10 +++++----- module/plugins/internal/XFSPCrypter.py | 9 +++++---- module/plugins/internal/XFSPHoster.py | 13 +++++++------ 8 files changed, 47 insertions(+), 40 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 760ed61b1..9267f4c2e 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -4,15 +4,16 @@ from module.plugins.Crypter import Crypter as _Crypter class DeadCrypter(_Crypter): - __name__ = "DeadCrypter" - __type__ = "crypter" + __name__ = "DeadCrypter" + __type__ = "crypter" __version__ = "0.02" __pattern__ = None + __config__ = [] __description__ = """ Crypter is no longer available """ - __license__ = "GPLv3" - __authors__ = [("stickell", "l.stickell@yahoo.it")] + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] def setup(self): diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index 2f0a01732..6633a8f1c 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -12,15 +12,16 @@ def create_getInfo(plugin): class DeadHoster(_Hoster): - __name__ = "DeadHoster" - __type__ = "hoster" + __name__ = "DeadHoster" + __type__ = "hoster" __version__ = "0.12" __pattern__ = None + __config__ = [] __description__ = """ Hoster is no longer available """ - __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] def setup(self): diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 5aadf6f2e..112736b0c 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -7,12 +7,15 @@ from module.utils import remove_chars class MultiHoster(Hook): - __name__ = "AbtractExtractor" + __name__ = "AbtractExtractor" + __type__ = "hook" __version__ = "0.19" + __config__ = [] + __description__ = """Generic MultiHoster plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] replacements = [("2shared.com", "twoshared.com"), ("4shared.com", "fourshared.com"), ("cloudnator.com", "shragle.com"), diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index f6783cffb..bfa2e6408 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -12,17 +12,18 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): - __name__ = "SimpleCrypter" - __type__ = "crypter" + __name__ = "SimpleCrypter" + __type__ = "crypter" __version__ = "0.20" __pattern__ = None + __config__ = [] __description__ = """Simple decrypter plugin""" - __license__ = "GPLv3" - __authors__ = [("stickell", "l.stickell@yahoo.it"), - ("zoidberg", "zoidberg@mujmail.cz"), - ("Walter Purcaro", "vuolter@gmail.com")] + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it"), + ("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] """ @@ -111,8 +112,6 @@ class SimpleCrypter(Crypter): if self.package_links: self.packages = [(package_name, self.package_links, folder_name)] - else: - self.fail(_("Could not extract any links")) def getLinks(self): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ca84dc33e..782655007 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -157,17 +157,18 @@ def timestamp(): class SimpleHoster(Hoster): - __name__ = "SimpleHoster" - __type__ = "hoster" + __name__ = "SimpleHoster" + __type__ = "hoster" __version__ = "0.42" __pattern__ = None + __config__ = [] __description__ = """Simple hoster plugin""" - __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("stickell", "l.stickell@yahoo.it"), - ("Walter Purcaro", "vuolter@gmail.com")] + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] """ diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index 69b5cfc22..574bdd920 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -10,14 +10,14 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSPAccount(Account): - __name__ = "XFSPAccount" - __type__ = "account" + __name__ = "XFSPAccount" + __type__ = "account" __version__ = "0.17" __description__ = """XFileSharingPro account plugin""" - __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("Walter Purcaro", "vuolter@gmail.com")] + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] """ diff --git a/module/plugins/internal/XFSPCrypter.py b/module/plugins/internal/XFSPCrypter.py index 372657d2d..52b6aed34 100644 --- a/module/plugins/internal/XFSPCrypter.py +++ b/module/plugins/internal/XFSPCrypter.py @@ -4,15 +4,16 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter class XFSPCrypter(SimpleCrypter): - __name__ = "XFSPCrypter" - __type__ = "crypter" + __name__ = "XFSPCrypter" + __type__ = "crypter" __version__ = "0.02" __pattern__ = None + __config__ = [] __description__ = """XFileSharingPro decrypter plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] HOSTER_NAME = None diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 44b2006ae..46529eaa8 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -15,17 +15,18 @@ from module.utils import html_unescape class XFSPHoster(SimpleHoster): - __name__ = "XFSPHoster" - __type__ = "hoster" + __name__ = "XFSPHoster" + __type__ = "hoster" __version__ = "0.06" __pattern__ = None + __config__ = [] __description__ = """XFileSharingPro hoster plugin""" - __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("stickell", "l.stickell@yahoo.it"), - ("Walter Purcaro", "vuolter@gmail.com")] + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] HOSTER_NAME = None -- cgit v1.2.3 From 483febe8e8711e644593ca758d1ba4c02c4dfde7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Oct 2014 23:10:50 +0100 Subject: [XFSPAccount] Improve VALID_UNTIL_PATTERN --- module/plugins/internal/XFSPAccount.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index 574bdd920..7b6215653 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.17" + __version__ = "0.18" __description__ = """XFileSharingPro account plugin""" __license__ = "GPLv3" @@ -34,7 +34,7 @@ class XFSPAccount(Account): COOKIES = [(HOSTER_NAME, "lang", "english")] #: or list of tuples [(domain, name, value)] - VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?([^><]+)' + VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?([^><]+)' TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+)\s*(?:(?P[\w^_]+)\s*)?' TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found @@ -64,7 +64,7 @@ class XFSPAccount(Account): m = re.search(self.VALID_UNTIL_PATTERN, html) if m: - expiredate = m.group(1) + expiredate = m.group(1).strip() self.logDebug("Expire date: " + expiredate) try: -- cgit v1.2.3 From c6c902698bb62f845e7be3156ec6b62d74200fec Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Oct 2014 23:15:36 +0100 Subject: [XFSPAccount] Fix missing gmtime import --- module/plugins/internal/XFSPAccount.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index 7b6215653..ec62e5c1e 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -3,7 +3,7 @@ import re from urlparse import urljoin -from time import mktime, strptime +from time import gmtime, mktime, strptime from module.plugins.Account import Account from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.18" + __version__ = "0.19" __description__ = """XFileSharingPro account plugin""" __license__ = "GPLv3" -- cgit v1.2.3 From 97cf4aee358bf95d5e77b5efd4e9e470c9631185 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 04:00:09 +0100 Subject: [XFSPAccount] Improve VALID_UNTIL_PATTERN 2 --- module/plugins/internal/XFSPAccount.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index ec62e5c1e..5f15a4659 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.19" + __version__ = "0.20" __description__ = """XFileSharingPro account plugin""" __license__ = "GPLv3" @@ -34,7 +34,7 @@ class XFSPAccount(Account): COOKIES = [(HOSTER_NAME, "lang", "english")] #: or list of tuples [(domain, name, value)] - VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?([^><]+)' + VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(\d{2} [\w^_]+ \d{4})' TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+)\s*(?:(?P[\w^_]+)\s*)?' TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found @@ -70,7 +70,7 @@ class XFSPAccount(Account): try: validuntil = mktime(strptime(expiredate, "%d %B %Y")) except Exception, e: - self.logError(e) + self.logError(str(e)) else: if validuntil > mktime(gmtime()): premium = True -- cgit v1.2.3 From 885f8ed782e64d9e73367905e642a84d0a8999f1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 04:01:38 +0100 Subject: Update __config__ --- module/plugins/internal/DeadCrypter.py | 1 - module/plugins/internal/DeadHoster.py | 1 - module/plugins/internal/MultiHoster.py | 2 -- module/plugins/internal/SimpleCrypter.py | 3 ++- module/plugins/internal/SimpleHoster.py | 1 - module/plugins/internal/XFSPCrypter.py | 1 - module/plugins/internal/XFSPHoster.py | 1 - 7 files changed, 2 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/DeadCrypter.py b/module/plugins/internal/DeadCrypter.py index 9267f4c2e..91cdf6036 100644 --- a/module/plugins/internal/DeadCrypter.py +++ b/module/plugins/internal/DeadCrypter.py @@ -9,7 +9,6 @@ class DeadCrypter(_Crypter): __version__ = "0.02" __pattern__ = None - __config__ = [] __description__ = """ Crypter is no longer available """ __license__ = "GPLv3" diff --git a/module/plugins/internal/DeadHoster.py b/module/plugins/internal/DeadHoster.py index 6633a8f1c..2d91b1d2d 100644 --- a/module/plugins/internal/DeadHoster.py +++ b/module/plugins/internal/DeadHoster.py @@ -17,7 +17,6 @@ class DeadHoster(_Hoster): __version__ = "0.12" __pattern__ = None - __config__ = [] __description__ = """ Hoster is no longer available """ __license__ = "GPLv3" diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 112736b0c..5c787e2d6 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -11,8 +11,6 @@ class MultiHoster(Hook): __type__ = "hook" __version__ = "0.19" - __config__ = [] - __description__ = """Generic MultiHoster plugin""" __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index bfa2e6408..50d647dee 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -17,7 +17,8 @@ class SimpleCrypter(Crypter): __version__ = "0.20" __pattern__ = None - __config__ = [] + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] __description__ = """Simple decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 782655007..3d2bcda86 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -162,7 +162,6 @@ class SimpleHoster(Hoster): __version__ = "0.42" __pattern__ = None - __config__ = [] __description__ = """Simple hoster plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/XFSPCrypter.py b/module/plugins/internal/XFSPCrypter.py index 52b6aed34..378b78d02 100644 --- a/module/plugins/internal/XFSPCrypter.py +++ b/module/plugins/internal/XFSPCrypter.py @@ -9,7 +9,6 @@ class XFSPCrypter(SimpleCrypter): __version__ = "0.02" __pattern__ = None - __config__ = [] __description__ = """XFileSharingPro decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 46529eaa8..5452b9444 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -20,7 +20,6 @@ class XFSPHoster(SimpleHoster): __version__ = "0.06" __pattern__ = None - __config__ = [] __description__ = """XFileSharingPro hoster plugin""" __license__ = "GPLv3" -- cgit v1.2.3 From 6a5b13502648cf51007c15948b778e5a9d54ad20 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 04:52:39 +0100 Subject: [SimpleCrypter][SimpleHoster] Fix typo in _error method call --- module/plugins/internal/SimpleCrypter.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 50d647dee..2fa0c2df3 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -14,7 +14,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.20" + __version__ = "0.21" __pattern__ = None __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -165,4 +165,4 @@ class SimpleCrypter(Crypter): def error(self, reason="", type="parse"): - return _error(reason, type) + return _error(self, reason, type) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 3d2bcda86..e0b31416a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -159,7 +159,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.42" + __version__ = "0.43" __pattern__ = None @@ -348,4 +348,4 @@ class SimpleHoster(Hoster): def error(self, reason="", type="parse"): - return _error(reason, type) + return _error(self, reason, type) -- cgit v1.2.3 From 302616d94f76ab8794b58d3d54b780cc58e86b06 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 04:58:47 +0100 Subject: Code cosmetics: items() -> iteritems() --- module/plugins/internal/MultiHoster.py | 2 +- module/plugins/internal/SimpleHoster.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 5c787e2d6..3596641b0 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -126,7 +126,7 @@ class MultiHoster(Hook): for name in self.core.pluginManager.hosterPlugins.keys(): pluginMap[name.lower()] = name - accountList = [name.lower() for name, data in self.core.accountManager.accounts.items() if data] + accountList = [name.lower() for name, data in self.core.accountManager.accounts.iteritems() if data] excludedList = [] for hoster in self.getHosterCached(): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e0b31416a..9e270ff3d 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -62,7 +62,7 @@ def parseHtmlForm(attr_str, html, input_names=None): if isinstance(input_names, dict): # check input attributes - for key, val in input_names.items(): + for key, val in input_names.iteritems(): if key in inputs: if isinstance(val, basestring) and inputs[key] == val: continue -- cgit v1.2.3 From 1a6d4ab20645e8dda3f0c61d10ecbcc16312eec7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 16:02:14 +0100 Subject: [SimpleCrypter] Wait method --- module/plugins/internal/SimpleCrypter.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 2fa0c2df3..1f41a7e5a 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -7,14 +7,14 @@ from traceback import print_exc from module.network.RequestFactory import getURL from module.plugins.Crypter import Crypter from module.plugins.Plugin import Fail -from module.plugins.internal.SimpleHoster import _error, replace_patterns, set_cookies +from module.plugins.internal.SimpleHoster import _error, _wait, replace_patterns, set_cookies from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.21" + __version__ = "0.22" __pattern__ = None __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -164,5 +164,10 @@ class SimpleCrypter(Crypter): self.package_links += self.getLinks() + #@TODO: Remove in 0.4.10 + def wait(self, seconds=0, reconnect=None): + return _wait(self, seconds, reconnect) + + def error(self, reason="", type="parse"): return _error(self, reason, type) -- cgit v1.2.3 From 8447df36b147bab1c3caa70aaa67732115ad735b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 16:03:14 +0100 Subject: [SimpleHoster] Improve wait method --- module/plugins/internal/SimpleHoster.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 9e270ff3d..c47a715f0 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -14,7 +14,7 @@ from module.utils import fixup, html_unescape, parseFileSize #@TODO: Remove in 0.4.10 and redirect to self.error instead -def _error(self, reason="", type=""): +def _error(self, reason, type): if not reason and not type: type = "unknown" @@ -27,6 +27,13 @@ def _error(self, reason="", type=""): raise Fail(msg) +#@TODO: Remove in 0.4.10 +def _wait(self, seconds, reconnect): + if seconds: + self.setWait(seconds, reconnect) + super(SimpleHoster, self).wait() + + def replace_patterns(string, ruleslist): for r in ruleslist: rf, rt = r @@ -159,7 +166,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.43" + __version__ = "0.44" __pattern__ = None @@ -341,10 +348,8 @@ class SimpleHoster(Hoster): #@TODO: Remove in 0.4.10 - def wait(self, seconds=False, reconnect=False): - if seconds: - self.setWait(seconds, reconnect) - super(SimpleHoster, self).wait() + def wait(self, seconds=0, reconnect=None): + return _wait(self, seconds, reconnect) def error(self, reason="", type="parse"): -- cgit v1.2.3 From 8b3589dd394d81177bf4680dddb5bdb9506b89ea Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 16:04:10 +0100 Subject: Update plugins to last changes --- module/plugins/internal/XFSPHoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 5452b9444..9e14fffbc 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -229,8 +229,8 @@ class XFSPHoster(SimpleHoster): elif 'premium' in self.errmsg and 'require' in self.errmsg: self.fail(_("File can be downloaded by premium users only")) elif 'limit' in self.errmsg: - self.wantReconnect = True - self.retry(25, 1 * 60 * 60, "Download limit exceeded") + self.wait(1 * 60 * 60, True) + self.retry(25, reason="Download limit exceeded") elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: self.retry(reason=_("Link expired")) elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: @@ -275,7 +275,7 @@ class XFSPHoster(SimpleHoster): if not self.premium: m = re.search(self.WAIT_PATTERN, self.html) if m: - wait_time = int(m.group(1)) + 1 + wait_time = int(m.group(1)) self.setWait(wait_time, False) else: wait_time = 0 -- cgit v1.2.3 From 34984dae733c3f3d47b41a0acfba3724d53c65a1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 16:52:10 +0100 Subject: Code cosmetics: plugin class attributes --- module/plugins/internal/AbstractExtractor.py | 6 +++--- module/plugins/internal/CaptchaService.py | 24 ++++++++++++------------ module/plugins/internal/UnRar.py | 6 +++--- module/plugins/internal/UnZip.py | 6 +++--- 4 files changed, 21 insertions(+), 21 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index 8282e6ff5..8a69ebb56 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -13,12 +13,12 @@ class WrongPassword(Exception): class AbtractExtractor: - __name__ = "AbtractExtractor" + __name__ = "AbtractExtractor" __version__ = "0.1" __description__ = """Abtract extractor plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] @staticmethod diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index aa8758344..2143e23df 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -6,12 +6,12 @@ from random import random class CaptchaService: - __name__ = "CaptchaService" + __name__ = "CaptchaService" __version__ = "0.14" __description__ = """Base captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] KEY_PATTERN = None @@ -51,12 +51,12 @@ class CaptchaService: class ReCaptcha(CaptchaService): - __name__ = "ReCaptcha" + __name__ = "ReCaptcha" __version__ = "0.07" __description__ = """ReCaptcha captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] KEY_PATTERN = r'recaptcha(/api|\.net)/(challenge|noscript)\?k=(?P[\w-]+)' @@ -110,12 +110,12 @@ class ReCaptcha(CaptchaService): class AdsCaptcha(CaptchaService): - __name__ = "AdsCaptcha" + __name__ = "AdsCaptcha" __version__ = "0.04" __description__ = """AdsCaptcha captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] ID_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*CaptchaId=(?P\d+)' @@ -172,12 +172,12 @@ class AdsCaptcha(CaptchaService): class SolveMedia(CaptchaService): - __name__ = "SolveMedia" + __name__ = "SolveMedia" __version__ = "0.05" __description__ = """SolveMedia captcha service plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(no)?script\?k=(?P.+?)["\']' diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index d8bb6f251..12bd84d20 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -21,12 +21,12 @@ def renice(pid, value): class UnRar(AbtractExtractor): - __name__ = "UnRar" + __name__ = "UnRar" __version__ = "0.18" __description__ = """Rar extractor plugin""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org")] + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org")] CMD = "unrar" diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index ec493bcd8..e754141a1 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -7,12 +7,12 @@ from module.plugins.internal.AbstractExtractor import AbtractExtractor class UnZip(AbtractExtractor): - __name__ = "UnZip" + __name__ = "UnZip" __version__ = "0.1" __description__ = """Zip extractor plugin""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org")] + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org")] @staticmethod -- cgit v1.2.3 From 227a0eb5157875e75a385118ec3485b044ccdb8a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 21:55:21 +0100 Subject: Improve loadAccountInfo stuff --- module/plugins/internal/XFSPAccount.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index 5f15a4659..db8f92aec 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.20" + __version__ = "0.21" __description__ = """XFileSharingPro account plugin""" __license__ = "GPLv3" @@ -27,7 +27,7 @@ class XFSPAccount(Account): example: HOSTER_URL = r'linestorage.com' PREMIUM_PATTERN: (optional) Checks if the account is premium - example: PREMIUM_PATTERN = r'>Renew premium<' + example: PREMIUM_PATTERN = r'>Renew premium' """ HOSTER_NAME = None @@ -74,7 +74,6 @@ class XFSPAccount(Account): else: if validuntil > mktime(gmtime()): premium = True - trafficleft = -1 else: if premium is False: #: registered account type (not premium) validuntil = -1 -- cgit v1.2.3 From 86e0fc7bfdb22f944b772e69c92bf7938d9a3117 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 30 Oct 2014 17:22:30 +0100 Subject: [XFSPHoster] Partially rewritten Now with ddl support --- module/plugins/internal/XFSPHoster.py | 107 +++++++++------------------------- 1 file changed, 28 insertions(+), 79 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py index 9e14fffbc..72a4a9d8c 100644 --- a/module/plugins/internal/XFSPHoster.py +++ b/module/plugins/internal/XFSPHoster.py @@ -4,10 +4,7 @@ import re from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME from random import random -from urllib import unquote -from urlparse import urlparse -from module.network.RequestFactory import getURL from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia from module.plugins.internal.SimpleHoster import create_getInfo, replace_patterns, set_cookies, SimpleHoster from module.plugins.Plugin import Fail @@ -17,7 +14,7 @@ from module.utils import html_unescape class XFSPHoster(SimpleHoster): __name__ = "XFSPHoster" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.07" __pattern__ = None @@ -68,80 +65,47 @@ class XFSPHoster(SimpleHoster): pattern = r'(https?://(www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]' self.LINK_PATTERN = pattern % self.HOSTER_NAME - if isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) - self.captcha = None self.errmsg = None self.passwords = self.getPassword().splitlines() - url = self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) - self.html = getURL(url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) - + return super(XFSPHoster, self).prepare() - def process(self, pyfile): - self.prepare() - - if not re.match(self.__pattern__, pyfile.url): - if self.premium: - self.handleOverriden() - else: - self.fail(_("Only premium users can download from other hosters with %s") % self.HOSTER_NAME) - else: - try: - self.file_info = self.getFileInfo() - except Fail: - self.file_info = None - - self.location = self.getDirectDownloadLink() - - if not self.file_info: - pyfile.name = html_unescape(unquote(urlparse( - self.location if self.location else pyfile.url).path.split("/")[-1])) - - if self.location: - self.startDownload(self.location) - elif self.premium: - self.logDebug("Handle as premium download") - self.handlePremium() - else: - self.logDebug("Handle as free download") - self.handleFree() + def handleFree(self): + link = self.getDownloadLink() - def getDirectDownloadLink(self): - """ Get download link for premium users with direct download enabled """ - self.req.http.lastURL = self.pyfile.url + if link: + if self.captcha: + self.correctCaptcha() - self.req.http.c.setopt(FOLLOWLOCATION, 0) - self.html = self.load(self.pyfile.url, decode=True) - self.header = self.req.http.header - self.req.http.c.setopt(FOLLOWLOCATION, 1) + self.download(link, ref=True, cookies=True, disposition=True) - location = None - m = re.search(r"Location\s*:\s*(.*)", self.header, re.I) - if m and re.match(self.LINK_PATTERN, m.group(1)): - location = m.group(1).strip() + elif self.errmsg and "captcha" in self.errmsg: + self.error(_("No valid captcha code entered")) - return location + else: + self.error(_("Download link not found")) - def handleFree(self): - url = self.getDownloadLink() - self.startDownload(url) + def handlePremium(self): + return self.handleFree() def getDownloadLink(self): for i in xrange(5): self.logDebug("Getting download link: #%d" % i) + data = self.getPostParameters() self.req.http.c.setopt(FOLLOWLOCATION, 0) + self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True) self.header = self.req.http.header + self.req.http.c.setopt(FOLLOWLOCATION, 1) - m = re.search(r"Location\s*:\s*(.*)", self.header, re.I) + m = re.search(r"Location\s*:\s*(.+)", self.header, re.I) if m: break @@ -149,23 +113,10 @@ class XFSPHoster(SimpleHoster): if m: break - else: - if self.errmsg and 'captcha' in self.errmsg: - self.fail(_("No valid captcha code entered")) - else: - self.fail(_("Download link not found")) - return m.group(1) - def handlePremium(self): - self.html = self.load(self.pyfile.url, post=self.getPostParameters()) - m = re.search(self.LINK_PATTERN, self.html) - if m is None: - self.error(_("LINK_PATTERN not found")) - self.startDownload(m.group(1)) - - + #@TODO: Re-enable def handleOverriden(self): #only tested with easybytez.com self.html = self.load("http://www.%s/" % self.HOSTER_NAME) @@ -199,21 +150,13 @@ class XFSPHoster(SimpleHoster): self.pyfile.url = m.group(1) - header = self.load(self.pyfile.url, just_header=True) + header = self.load(self.pyfile.url, just_header=True, decode=True) if 'location' in header: # Direct link - self.startDownload(header['location']) + self.download(header['location'], ref=True, cookies=True, disposition=True) else: self.retry(reason=_("OVR link location not found")) - def startDownload(self, link): - link = link.strip() - if self.captcha: - self.correctCaptcha() - self.logDebug("DIRECT LINK: %s" % link) - self.download(link, disposition=True) - - def checkErrors(self): m = re.search(self.ERROR_PATTERN, self.html) if m: @@ -224,22 +167,28 @@ class XFSPHoster(SimpleHoster): wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in re.findall(r'(\d+)\s*(hour|minute|second)', self.errmsg)]) self.wait(wait_time, True) + elif 'captcha' in self.errmsg: self.invalidCaptcha() + elif 'premium' in self.errmsg and 'require' in self.errmsg: self.fail(_("File can be downloaded by premium users only")) + elif 'limit' in self.errmsg: self.wait(1 * 60 * 60, True) self.retry(25, reason="Download limit exceeded") + elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: self.retry(reason=_("Link expired")) + elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: self.tempOffline() + elif 'download files up to' in self.errmsg: self.fail(_("File too large for free download")) + else: self.fail(self.errmsg) - else: self.errmsg = None -- cgit v1.2.3 From 71be42ba4635ef49c1cc7690be8805f00bc35e3c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 30 Oct 2014 17:23:18 +0100 Subject: [XFSPAccount] Fix and improve loadAccountInfo --- module/plugins/internal/XFSPAccount.py | 43 ++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 18 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py index db8f92aec..381e919e6 100644 --- a/module/plugins/internal/XFSPAccount.py +++ b/module/plugins/internal/XFSPAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSPAccount(Account): __name__ = "XFSPAccount" __type__ = "account" - __version__ = "0.21" + __version__ = "0.22" __description__ = """XFileSharingPro account plugin""" __license__ = "GPLv3" @@ -34,9 +34,9 @@ class XFSPAccount(Account): COOKIES = [(HOSTER_NAME, "lang", "english")] #: or list of tuples [(domain, name, value)] - VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(\d{2} [\w^_]+ \d{4})' + VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(\d{1,2} [\w^_]+ \d{4})' - TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+)\s*(?:(?P[\w^_]+)\s*)?' + TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' @@ -48,6 +48,9 @@ class XFSPAccount(Account): def init(self): + # if not self.HOSTER_NAME: + # self.fail(_("Missing HOSTER_NAME")) + if not hasattr(self, "HOSTER_URL"): self.HOSTER_URL = "http://www.%s/" % self.HOSTER_NAME.replace("www.", "", 1) @@ -79,22 +82,26 @@ class XFSPAccount(Account): validuntil = -1 premium = False - try: - traffic = re.search(self.TRAFFIC_LEFT_PATTERN, html).groupdict() - if "Unlimited" in traffic['S']: - trafficleft = -1 - if premium is None: - premium = True - else: - if 'U' in traffic: - unit = traffic['U'] - elif isinstance(self.TRAFFIC_LEFT_UNIT, basestring): - unit = self.TRAFFIC_LEFT_UNIT + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if m: + try: + traffic = m.groupdict() + if "nlimited" in traffic['S']: + trafficleft = -1 + if premium is None: + premium = True else: - unit = None - trafficleft = self.parseTraffic(traffic['S'], unit) - except: - pass + if 'U' in traffic: + unit = traffic['U'] + elif isinstance(self.TRAFFIC_LEFT_UNIT, basestring): + unit = self.TRAFFIC_LEFT_UNIT + else: + unit = "" + + trafficleft = self.parseTraffic(traffic['S'] + unit) + + except Exception, e: + self.logDebug(str(e)) return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium or False} -- cgit v1.2.3 From fe64575ff079037d0db827e9067d5a5f6dffb1dd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 30 Oct 2014 17:24:09 +0100 Subject: [SimpleCrypter] Fix cookies support --- module/plugins/internal/SimpleCrypter.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 1f41a7e5a..e586580d7 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -4,7 +4,6 @@ import re from traceback import print_exc -from module.network.RequestFactory import getURL from module.plugins.Crypter import Crypter from module.plugins.Plugin import Fail from module.plugins.internal.SimpleHoster import _error, _wait, replace_patterns, set_cookies @@ -14,7 +13,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.22" + __version__ = "0.23" __pattern__ = None __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -94,7 +93,7 @@ class SimpleCrypter(Crypter): set_cookies(self.req.cj, self.COOKIES) self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - self.html = getURL(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) def decrypt(self, pyfile): -- cgit v1.2.3 From 56ff5aff808975456163c027c670844baeebc2a8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 30 Oct 2014 17:16:09 +0100 Subject: [SimpleHoster] Fix checkTrafficLeft, ddl routine and cookies support --- module/plugins/internal/SimpleHoster.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index c47a715f0..de21f4d3c 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -166,7 +166,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.44" + __version__ = "0.45" __pattern__ = None @@ -233,19 +233,20 @@ class SimpleHoster(Hoster): self.req.setOption("timeout", 120) self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) - self.html = getURL(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) - - - def process(self, pyfile): - self.prepare() if self.CHECK_DIRECT_DOWNLOAD and self.premium: - header = self.load(self.pyfile.url, just_header=True) + header = self.load(self.pyfile.url, just_header=True, decode=True) if 'location' in header: self.logDebug("Direct download link detected") self.download(header['location'], ref=True, cookies=True, disposition=True) return + self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + + + def process(self, pyfile): + self.prepare() + premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) if not premium_only: #: Usually premium only pages doesn't show the file information self.getFileInfo() @@ -340,11 +341,15 @@ class SimpleHoster(Hoster): def checkTrafficLeft(self): traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] - if traffic == -1: + + if traffic is None: + return False + elif traffic == -1: return True - size = self.pyfile.size / 1024 - self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) - return size <= traffic + else: + size = self.pyfile.size / 1024 + self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) + return size <= traffic #@TODO: Remove in 0.4.10 -- cgit v1.2.3 From 7d21001f0495102a2948912d5416dbd3362dc446 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 1 Nov 2014 18:50:11 +0100 Subject: [SimpleCrypter] Move TEXT_ENCODING check to prepare routine --- module/plugins/internal/SimpleCrypter.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index e586580d7..86e8077e7 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -13,7 +13,7 @@ from module.utils import fixup, html_unescape class SimpleCrypter(Crypter): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.23" + __version__ = "0.24" __pattern__ = None __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -93,8 +93,12 @@ class SimpleCrypter(Crypter): set_cookies(self.req.cj, self.COOKIES) self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) + self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + if isinstance(self.TEXT_ENCODING, basestring): + self.html = unicode(self.html, self.TEXT_ENCODING) + def decrypt(self, pyfile): self.prepare() @@ -130,9 +134,6 @@ class SimpleCrypter(Crypter): def getPackageNameAndFolder(self): - if isinstance(self.TEXT_ENCODING, basestring): - self.html = unicode(self.html, self.TEXT_ENCODING) - if hasattr(self, 'TITLE_PATTERN'): try: m = re.search(self.TITLE_PATTERN, self.html) -- cgit v1.2.3 From 011c308fdce25df396824ea97c2bf634b976c8d8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 1 Nov 2014 18:51:19 +0100 Subject: [SimpleHoster] Fix and improve ddl detection, parseFileInfo and getInfo --- module/plugins/internal/SimpleHoster.py | 122 +++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 42 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index de21f4d3c..e601759cf 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -90,27 +90,30 @@ def parseHtmlForm(attr_str, html, input_names=None): def parseFileInfo(self, url="", html=""): - info = {"name": url, "size": 0, "status": 3} - - if hasattr(self, "pyfile"): + if not url and hasattr(self, "pyfile"): url = self.pyfile.url - if hasattr(self, "req") and self.req.http.code == '404': - info['status'] = 1 - else: - if not html and hasattr(self, "html"): - html = self.html - if isinstance(self.TEXT_ENCODING, basestring): - html = unicode(html, self.TEXT_ENCODING) - if hasattr(self, "html"): - self.html = html + info = {"name": url, "size": 0, "status": 3} + if not html: + if url: + return tuple(create_getInfo(self)([url])) + elif hasattr(self, "html"): + if hasattr(self, "req") and self.req.http.code == '404': + info['status'] = 1 + else: + html = self.html + + if html: if hasattr(self, "OFFLINE_PATTERN") and re.search(self.OFFLINE_PATTERN, html): info['status'] = 1 + elif hasattr(self, "FILE_OFFLINE_PATTERN") and re.search(self.FILE_OFFLINE_PATTERN, html): #@TODO: Remove in 0.4.10 info['status'] = 1 + elif hasattr(self, "TEMP_OFFLINE_PATTERN") and re.search(self.TEMP_OFFLINE_PATTERN, html): info['status'] = 6 + else: online = False try: @@ -128,18 +131,23 @@ def parseFileInfo(self, url="", html=""): if online: # File online, return name and size info['status'] = 2 + if 'N' in info: info['name'] = replace_patterns(info['N'].strip(), self.FILE_NAME_REPLACEMENTS) + if 'S' in info: size = replace_patterns(info['S'] + info['U'] if 'U' in info else info['S'], self.FILE_SIZE_REPLACEMENTS) info['size'] = parseFileSize(size) + elif isinstance(info['size'], basestring): unit = info['units'] if 'units' in info else None info['size'] = parseFileSize(info['size'], unit) - if hasattr(self, "file_info"): - self.file_info = info + if not hasattr(self, "file_info"): + self.file_info = {} + + self.file_info.update(info) return info['name'], info['size'], info['status'], url @@ -148,13 +156,23 @@ def create_getInfo(plugin): def getInfo(urls): for url in urls: - cj = CookieJar(plugin.__name__) - if isinstance(plugin.COOKIES, list): + if hasattr(plugin, "COOKIES") and isinstance(plugin.COOKIES, list): + cj = CookieJar(plugin.__name__) set_cookies(cj, plugin.COOKIES) + else: + cj = None - file_info = parseFileInfo(plugin, url, getURL(replace_patterns(url, plugin.FILE_URL_REPLACEMENTS), - decode=not plugin.TEXT_ENCODING, cookies=cj)) - yield file_info + if hasattr(plugin, "FILE_URL_REPLACEMENTS"): + url = replace_patterns(url, plugin.FILE_URL_REPLACEMENTS) + + if hasattr(plugin, "TEXT_ENCODING"): + html = getURL(url, cookies=bool(cj), decode=not plugin.TEXT_ENCODING) + if isinstance(plugin.TEXT_ENCODING, basestring): + html = unicode(html, plugin.TEXT_ENCODING) + else: + html = getURL(url, cookies=bool(cj), decode=True) + + yield parseFileInfo(plugin, url, html) return getInfo @@ -166,7 +184,7 @@ def timestamp(): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.45" + __version__ = "0.46" __pattern__ = None @@ -215,11 +233,10 @@ class SimpleHoster(Hoster): COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - CHECK_DIRECT_DOWNLOAD = True #: Search for a direct download link in the page header (premium only) - def init(self): self.file_info = {} + self.html = "" #@TODO: Remove in 0.4.10 def setup(self): @@ -234,31 +251,53 @@ class SimpleHoster(Hoster): self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS) - if self.CHECK_DIRECT_DOWNLOAD and self.premium: - header = self.load(self.pyfile.url, just_header=True, decode=True) - if 'location' in header: - self.logDebug("Direct download link detected") - self.download(header['location'], ref=True, cookies=True, disposition=True) - return + if self.premium: + direct_link = self.getDirectLink(self.pyfile.url) + if direct_link: + return direct_link self.html = self.load(self.pyfile.url, decode=not self.TEXT_ENCODING, cookies=bool(self.COOKIES)) + if isinstance(self.TEXT_ENCODING, basestring): + self.html = unicode(self.html, self.TEXT_ENCODING) + def process(self, pyfile): - self.prepare() + direct_link = self.prepare() - premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only: #: Usually premium only pages doesn't show the file information - self.getFileInfo() + if isinstance(direct_link, basestring): + self.download(direct_link, ref=True, cookies=True, disposition=True) + + elif self.html is None: + self.fail(_("Attribute html should never be set to None")) - if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): - self.logDebug("Handle as premium download") - self.handlePremium() - elif premium_only: - self.fail(_("This link require a premium account")) else: - self.logDebug("Handle as free download") - self.handleFree() + premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) + if not premium_only and 'status' not in self.file_info: #: Usually premium only pages doesn't show the file information + self.getFileInfo() + + if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as premium download") + self.handlePremium() + elif premium_only: + self.fail(_("Link require a premium account to be handled")) + else: + self.logDebug("Handled as free download") + self.handleFree() + + + def getDirectLink(self, url): + self.req.http.c.setopt(FOLLOWLOCATION, 0) + + html = self.load(url, ref=True, decode=True) + + self.req.http.c.setopt(FOLLOWLOCATION, 1) + + if parseFileInfo(self, url, html)[2] != 2: + try + return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1) + except: + pass def getFileInfo(self): @@ -271,8 +310,7 @@ class SimpleHoster(Hoster): elif status == 6: self.tempOffline() elif status != 2: - self.logDebug(self.file_info) - self.error(_("File info")) + self.error(_("File info: %s") % self.file_info) if name: self.pyfile.name = name @@ -284,7 +322,7 @@ class SimpleHoster(Hoster): else: self.logError(_("File size not parsed")) - self.logDebug("FILE NAME: %s FILE SIZE: %s" % (self.pyfile.name, self.pyfile.size)) + self.logDebug("FILE NAME: %s" % self.pyfile.name, "FILE SIZE: %d" % self.pyfile.size) return self.file_info -- cgit v1.2.3 From 11fcb4ebb1d3f38e8a49f316308520ebcf8cf658 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 1 Nov 2014 22:20:06 +0100 Subject: [SimpleHoster] Fix typo --- module/plugins/internal/SimpleHoster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index e601759cf..1d85dc01b 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -294,7 +294,7 @@ class SimpleHoster(Hoster): self.req.http.c.setopt(FOLLOWLOCATION, 1) if parseFileInfo(self, url, html)[2] != 2: - try + try: return re.search(r"Location\s*:\s*(.+)", self.req.http.header, re.I).group(1) except: pass -- cgit v1.2.3 From 7e759ef2d5dbb30da9752b72146bf4ff240821a8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Nov 2014 22:34:38 +0100 Subject: Update internal XFS plugins --- module/plugins/internal/XFSAccount.py | 127 ++++++++++++++ module/plugins/internal/XFSCrypter.py | 29 ++++ module/plugins/internal/XFSHoster.py | 299 +++++++++++++++++++++++++++++++++ module/plugins/internal/XFSPAccount.py | 127 -------------- module/plugins/internal/XFSPCrypter.py | 28 --- module/plugins/internal/XFSPHoster.py | 297 -------------------------------- 6 files changed, 455 insertions(+), 452 deletions(-) create mode 100644 module/plugins/internal/XFSAccount.py create mode 100644 module/plugins/internal/XFSCrypter.py create mode 100644 module/plugins/internal/XFSHoster.py delete mode 100644 module/plugins/internal/XFSPAccount.py delete mode 100644 module/plugins/internal/XFSPCrypter.py delete mode 100644 module/plugins/internal/XFSPHoster.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py new file mode 100644 index 000000000..597964d4c --- /dev/null +++ b/module/plugins/internal/XFSAccount.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin +from time import gmtime, mktime, strptime + +from module.plugins.Account import Account +from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies + + +class XFSAccount(Account): + __name__ = "XFSAccount" + __type__ = "account" + __version__ = "0.23" + + __description__ = """XFileSharing account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + """ + Following patterns should be defined by each hoster: + + HOSTER_URL: (optional) + example: HOSTER_URL = r'linestorage.com' + + PREMIUM_PATTERN: (optional) Checks if the account is premium + example: PREMIUM_PATTERN = r'>Renew premium' + """ + + HOSTER_DOMAIN = None + + COOKIES = [(HOSTER_DOMAIN, "lang", "english")] #: or list of tuples [(domain, name, value)] + + VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(\d{1,2} [\w^_]+ \d{4})' + + TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' + TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found + + LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' + + + def __init__(self, manager, accounts): #@TODO: remove in 0.4.10 + self.init() + return super(XFSAccount, self).__init__(manager, accounts) + + + def init(self): + # if not self.HOSTER_DOMAIN: + # self.fail(_("Missing HOSTER_DOMAIN")) + + if not hasattr(self, "HOSTER_URL"): + self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN.replace("www.", "", 1) + + + def loadAccountInfo(self, user, req): + html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) + + validuntil = None + trafficleft = None + premium = None + + if hasattr(self, "PREMIUM_PATTERN"): + premium = True if re.search(self.PREMIUM_PATTERN, html) else False + + m = re.search(self.VALID_UNTIL_PATTERN, html) + if m: + expiredate = m.group(1).strip() + self.logDebug("Expire date: " + expiredate) + + try: + validuntil = mktime(strptime(expiredate, "%d %B %Y")) + except Exception, e: + self.logError(str(e)) + else: + if validuntil > mktime(gmtime()): + premium = True + else: + if premium is False: #: registered account type (not premium) + validuntil = -1 + premium = False + + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if m: + try: + traffic = m.groupdict() + if "nlimited" in traffic['S']: + trafficleft = -1 + if premium is None: + premium = True + else: + if 'U' in traffic: + unit = traffic['U'] + elif isinstance(self.TRAFFIC_LEFT_UNIT, basestring): + unit = self.TRAFFIC_LEFT_UNIT + else: + unit = "" + + trafficleft = self.parseTraffic(traffic['S'] + unit) + + except Exception, e: + self.logDebug(str(e)) + + return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium or False} + + + def login(self, user, data, req): + if isinstance(self.COOKIES, list): + set_cookies(req.cj, self.COOKIES) + + url = urljoin(self.HOSTER_URL, "login.html") + html = req.load(url, decode=True) + + action, inputs = parseHtmlForm('name="FL"', html) + if not inputs: + inputs = {'op': "login", + 'redirect': self.HOSTER_URL} + + inputs.update({'login': user, + 'password': data['password']}) + + html = req.load(self.HOSTER_URL, post=inputs, decode=True) + + if re.search(self.LOGIN_FAIL_PATTERN, html): + self.wrongPassword() diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py new file mode 100644 index 000000000..5a126707b --- /dev/null +++ b/module/plugins/internal/XFSCrypter.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +from module.plugins.internal.SimpleCrypter import SimpleCrypter + + +class XFSCrypter(SimpleCrypter): + __name__ = "XFSCrypter" + __type__ = "crypter" + __version__ = "0.03" + + __pattern__ = None + + __description__ = """XFileSharing decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = None + HOSTER_NAME = None + + URL_REPLACEMENTS = [(r'[?/&]+$', r''), (r'(.+/[^?]*)$', r'\1?'), (r'$', r'&per_page=10000')] + + COOKIES = [(HOSTER_DOMAIN, "lang", "english")] + + LINK_PATTERN = r'<(?:td|TD) [^>]*>\s*]*>.+?(?:)?\s*' + NAME_PATTERN = r'<[tT]itle>.*?\: (.+) folder' + + OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py new file mode 100644 index 000000000..b7a407a19 --- /dev/null +++ b/module/plugins/internal/XFSHoster.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- + +import re + +from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME +from random import random + +from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia +from module.plugins.internal.SimpleHoster import create_getInfo, replace_patterns, set_cookies, SimpleHoster +from module.plugins.Plugin import Fail +from module.utils import html_unescape + + +class XFSHoster(SimpleHoster): + __name__ = "XFSHoster" + __type__ = "hoster" + __version__ = "0.08" + + __pattern__ = None + + __description__ = """XFileSharing hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = None + HOSTER_NAME = None + + COOKIES = [(HOSTER_DOMAIN, "lang", "english")] + + INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' + NAME_PATTERN = r'[\d\.\,]+) ?(?P[\w^_]+)?\)' + + OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' + + WAIT_PATTERN = r'.*?>(\d+)' + + OVR_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' + LINK_PATTERN = None #: final download url pattern + + CAPTCHA_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' + CAPTCHA_DIV_PATTERN = r'>Enter code.*?(.+?)
' + RECAPTCHA_PATTERN = None + SOLVEMEDIA_PATTERN = None + + ERROR_PATTERN = r'(?:class=["\']err["\'][^>]*>|<[Cc]enter>)(.+?)(?:["\']|", " ", self.errmsg)) + + if 'wait' in self.errmsg: + wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in + re.findall(r'(\d+)\s*(hour|minute|second)', self.errmsg)]) + self.wait(wait_time, True) + + elif 'captcha' in self.errmsg: + self.invalidCaptcha() + + elif 'premium' in self.errmsg and 'require' in self.errmsg: + self.fail(_("File can be downloaded by premium users only")) + + elif 'limit' in self.errmsg: + self.wait(1 * 60 * 60, True) + self.retry(25, reason="Download limit exceeded") + + elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: + self.retry(reason=_("Link expired")) + + elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: + self.tempOffline() + + elif 'download files up to' in self.errmsg: + self.fail(_("File too large for free download")) + + else: + self.fail(self.errmsg) + else: + self.errmsg = None + + return self.errmsg + + + def getPostParameters(self): + for _i in xrange(3): + self.checkErrors() + + if hasattr(self, "FORM_PATTERN"): + action, inputs = self.parseHtmlForm(self.FORM_PATTERN) + else: + action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) + + if not inputs: + action, inputs = self.parseHtmlForm('F1') + if not inputs: + if self.errmsg: + self.retry(reason=self.errmsg) + else: + self.error(_("Form not found")) + + self.logDebug(inputs) + + if 'op' in inputs and inputs['op'] in ("download2", "download3"): + if "password" in inputs: + if self.passwords: + inputs['password'] = self.passwords.pop(0) + else: + self.fail(_("No or invalid passport")) + + if not self.premium: + m = re.search(self.WAIT_PATTERN, self.html) + if m: + wait_time = int(m.group(1)) + self.setWait(wait_time, False) + else: + wait_time = 0 + + self.captcha = self.handleCaptcha(inputs) + + if wait_time: + self.wait() + + self.errmsg = None + return inputs + + else: + inputs['referer'] = self.pyfile.url + + if self.premium: + inputs['method_premium'] = "Premium Download" + if 'method_free' in inputs: + del inputs['method_free'] + else: + inputs['method_free'] = "Free Download" + if 'method_premium' in inputs: + del inputs['method_premium'] + + self.html = self.load(self.pyfile.url, post=inputs, ref=True) + self.errmsg = None + + else: + self.error(_("FORM: %s") % (inputs['op'] if 'op' in inputs else _("UNKNOWN"))) + + + def handleCaptcha(self, inputs): + m = re.search(self.CAPTCHA_PATTERN, self.html) + if m: + captcha_url = m.group(1) + inputs['code'] = self.decryptCaptcha(captcha_url) + return 1 + + m = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.S) + if m: + captcha_div = m.group(1) + self.logDebug(captcha_div) + numerals = re.findall(r'(\d)', html_unescape(captcha_div)) + inputs['code'] = "".join([a[1] for a in sorted(numerals, key=lambda num: int(num[0]))]) + self.logDebug("Captcha code: %s" % inputs['code'], numerals) + return 2 + + recaptcha = ReCaptcha(self) + try: + captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) + except: + captcha_key = recaptcha.detect_key() + + if captcha_key: + self.logDebug("ReCaptcha key: %s" % captcha_key) + inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(captcha_key) + return 3 + + solvemedia = SolveMedia(self) + try: + captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) + except: + captcha_key = solvemedia.detect_key() + + if captcha_key: + self.logDebug("SolveMedia key: %s" % captcha_key) + inputs['adcopy_challenge'], inputs['adcopy_response'] = solvemedia.challenge(captcha_key) + return 4 + + return 0 diff --git a/module/plugins/internal/XFSPAccount.py b/module/plugins/internal/XFSPAccount.py deleted file mode 100644 index 381e919e6..000000000 --- a/module/plugins/internal/XFSPAccount.py +++ /dev/null @@ -1,127 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from urlparse import urljoin -from time import gmtime, mktime, strptime - -from module.plugins.Account import Account -from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies - - -class XFSPAccount(Account): - __name__ = "XFSPAccount" - __type__ = "account" - __version__ = "0.22" - - __description__ = """XFileSharingPro account plugin""" - __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("Walter Purcaro", "vuolter@gmail.com")] - - - """ - Following patterns should be defined by each hoster: - - HOSTER_URL: (optional) - example: HOSTER_URL = r'linestorage.com' - - PREMIUM_PATTERN: (optional) Checks if the account is premium - example: PREMIUM_PATTERN = r'>Renew premium' - """ - - HOSTER_NAME = None - - COOKIES = [(HOSTER_NAME, "lang", "english")] #: or list of tuples [(domain, name, value)] - - VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(\d{1,2} [\w^_]+ \d{4})' - - TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' - TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found - - LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' - - - def __init__(self, manager, accounts): #@TODO: remove in 0.4.10 - self.init() - return super(XFSPAccount, self).__init__(manager, accounts) - - - def init(self): - # if not self.HOSTER_NAME: - # self.fail(_("Missing HOSTER_NAME")) - - if not hasattr(self, "HOSTER_URL"): - self.HOSTER_URL = "http://www.%s/" % self.HOSTER_NAME.replace("www.", "", 1) - - - def loadAccountInfo(self, user, req): - html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) - - validuntil = None - trafficleft = None - premium = None - - if hasattr(self, "PREMIUM_PATTERN"): - premium = True if re.search(self.PREMIUM_PATTERN, html) else False - - m = re.search(self.VALID_UNTIL_PATTERN, html) - if m: - expiredate = m.group(1).strip() - self.logDebug("Expire date: " + expiredate) - - try: - validuntil = mktime(strptime(expiredate, "%d %B %Y")) - except Exception, e: - self.logError(str(e)) - else: - if validuntil > mktime(gmtime()): - premium = True - else: - if premium is False: #: registered account type (not premium) - validuntil = -1 - premium = False - - m = re.search(self.TRAFFIC_LEFT_PATTERN, html) - if m: - try: - traffic = m.groupdict() - if "nlimited" in traffic['S']: - trafficleft = -1 - if premium is None: - premium = True - else: - if 'U' in traffic: - unit = traffic['U'] - elif isinstance(self.TRAFFIC_LEFT_UNIT, basestring): - unit = self.TRAFFIC_LEFT_UNIT - else: - unit = "" - - trafficleft = self.parseTraffic(traffic['S'] + unit) - - except Exception, e: - self.logDebug(str(e)) - - return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium or False} - - - def login(self, user, data, req): - if isinstance(self.COOKIES, list): - set_cookies(req.cj, self.COOKIES) - - url = urljoin(self.HOSTER_URL, "login.html") - html = req.load(url, decode=True) - - action, inputs = parseHtmlForm('name="FL"', html) - if not inputs: - inputs = {'op': "login", - 'redirect': self.HOSTER_URL} - - inputs.update({'login': user, - 'password': data['password']}) - - html = req.load(self.HOSTER_URL, post=inputs, decode=True) - - if re.search(self.LOGIN_FAIL_PATTERN, html): - self.wrongPassword() diff --git a/module/plugins/internal/XFSPCrypter.py b/module/plugins/internal/XFSPCrypter.py deleted file mode 100644 index 378b78d02..000000000 --- a/module/plugins/internal/XFSPCrypter.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.plugins.internal.SimpleCrypter import SimpleCrypter - - -class XFSPCrypter(SimpleCrypter): - __name__ = "XFSPCrypter" - __type__ = "crypter" - __version__ = "0.02" - - __pattern__ = None - - __description__ = """XFileSharingPro decrypter plugin""" - __license__ = "GPLv3" - __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - - - HOSTER_NAME = None - - URL_REPLACEMENTS = [(r'[?/&]+$', r''), (r'(.+/[^?]*)$', r'\1?'), (r'$', r'&per_page=10000')] - - COOKIES = [(HOSTER_NAME, "lang", "english")] - - LINK_PATTERN = r'<(?:td|TD) [^>]*>\s*]*>.+?(?:)?\s*' - TITLE_PATTERN = r'<[tT]itle>.*?\: (.+) folder' - - OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' - TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' diff --git a/module/plugins/internal/XFSPHoster.py b/module/plugins/internal/XFSPHoster.py deleted file mode 100644 index 72a4a9d8c..000000000 --- a/module/plugins/internal/XFSPHoster.py +++ /dev/null @@ -1,297 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME -from random import random - -from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia -from module.plugins.internal.SimpleHoster import create_getInfo, replace_patterns, set_cookies, SimpleHoster -from module.plugins.Plugin import Fail -from module.utils import html_unescape - - -class XFSPHoster(SimpleHoster): - __name__ = "XFSPHoster" - __type__ = "hoster" - __version__ = "0.07" - - __pattern__ = None - - __description__ = """XFileSharingPro hoster plugin""" - __license__ = "GPLv3" - __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), - ("stickell", "l.stickell@yahoo.it"), - ("Walter Purcaro", "vuolter@gmail.com")] - - - HOSTER_NAME = None - - FILE_URL_REPLACEMENTS = [] - - COOKIES = [(HOSTER_NAME, "lang", "english")] - - FILE_INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' - FILE_NAME_PATTERN = r'[\d\.\,]+) ?(?P[\w^_]+)?\)' - - OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' - TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' - - WAIT_PATTERN = r'.*?>(\d+)' - - OVR_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' - LINK_PATTERN = None #: final download url pattern - - CAPTCHA_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' - CAPTCHA_DIV_PATTERN = r'>Enter code.*?(.+?)
' + RECAPTCHA_PATTERN = None + SOLVEMEDIA_PATTERN = None FORM_PATTERN = None FORM_INPUTS_MAP = None #: dict passed as input_names to parseHtmlForm @@ -234,10 +234,10 @@ class XFSHoster(SimpleHoster): retries = 3 else: delay = 1 * 60 * 60 - retries = 25 + retries = 24 - self.wait(delay, True) - self.retry(retries, reason=_("Download limit exceeded")) + self.wantReconnect = True + self.retry(retries, delay, _("Download limit exceeded")) elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: self.retry(reason=_("Link expired")) @@ -249,6 +249,7 @@ class XFSHoster(SimpleHoster): self.fail(_("File too large for free download")) else: + self.wantReconnect = True self.retry(wait_time=60, reason=self.errmsg) if self.errmsg: @@ -311,7 +312,7 @@ class XFSHoster(SimpleHoster): inputs['code'] = self.decryptCaptcha(captcha_url) return 1 - m = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.S) + m = re.search(self.CAPTCHA_BLOCK_PATTERN, self.html, re.S) if m: captcha_div = m.group(1) numerals = re.findall(r'(\d)', html_unescape(captcha_div)) -- cgit v1.2.3 From a83f45f14aefb4fd766f7908e11bc6beaac3d0e8 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 1 Dec 2014 18:51:37 +0100 Subject: [XFSAccount] Improve patterns --- module/plugins/internal/XFSAccount.py | 8 ++++---- module/plugins/internal/XFSHoster.py | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 5a265c08a..2094b1480 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.31" + __version__ = "0.32" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -27,15 +27,15 @@ class XFSAccount(Account): PREMIUM_PATTERN = r'\(Premium only\)' - VALID_UNTIL_PATTERN = r'>Premium.[Aa]ccount expire:.*?(\d{1,2} [\w^_]+ \d{4})' + VALID_UNTIL_PATTERN = r'Premium.[Aa]ccount expire:.*?(\d{1,2} [\w^_]+ \d{4})' - TRAFFIC_LEFT_PATTERN = r'>Traffic available today:.*?\s*(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' + TRAFFIC_LEFT_PATTERN = r'Traffic available today:.*?\s*(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found LEECH_TRAFFIC_PATTERN = r'Leech Traffic left:.*?(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' LEECH_TRAFFIC_UNIT = "MB" #: used only if no group was found - LOGIN_FAIL_PATTERN = r'>(Incorrect Login or Password|Error<)' + LOGIN_FAIL_PATTERN = r'>\s*(Incorrect Login or Password|Error<)' def __init__(self, manager, accounts): #@TODO: remove in 0.4.10 diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 587d47f6e..c3db3f335 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -35,7 +35,6 @@ class XFSHoster(SimpleHoster): CHECK_DIRECT_LINK = None MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... - INFO_PATTERN = r'Filename:(?P[^<]+)\s*.*?\((?P[^<]+)\)' NAME_PATTERN = r'(>Filename:|name="fname" value="|)(?P.+?)(\s*<|")' SIZE_PATTERN = r'(>Size:|>File:.*>|)(?P[\d.,]+)\s*(?P[\w^_]+)' -- cgit v1.2.3 From 1bcdfb409ded0c369735c413b33f2e344b36bd7a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 1 Dec 2014 20:00:14 +0100 Subject: Fixes --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 736d15f3a..cf03a1242 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -123,7 +123,7 @@ def _isDirectLink(self, url, resumable=True): location = header['location'] if resumable: #: sometimes http code may be wrong... - if self.load(location, ref=True, cookies=True, just_header=True, decode=True)['location']: + if 'location' in self.load(location, ref=True, cookies=True, just_header=True, decode=True): return "" else: if not 'code' in header or header['code'] != 302: @@ -388,7 +388,7 @@ class SimpleHoster(Hoster): self.checkFile() - def checkFile(self) + def checkFile(self): if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": #@TODO: Move to hoster in 0.4.10 self.fail(_("Empty file")) -- cgit v1.2.3 From eb71f308010e35aaec7f7bfa529ada96975c909d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 2 Dec 2014 00:04:41 +0100 Subject: [SimpleHoster] Force _isDirectLink to old style --- module/plugins/internal/SimpleHoster.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index cf03a1242..ba718950d 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -122,6 +122,8 @@ def _isDirectLink(self, url, resumable=True): location = header['location'] + resumable = False #@NOTE: Testing... + if resumable: #: sometimes http code may be wrong... if 'location' in self.load(location, ref=True, cookies=True, just_header=True, decode=True): return "" -- cgit v1.2.3 From be05dce2da77bca4613efbdfb0f6357a983e6e1c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 3 Dec 2014 14:38:22 +0100 Subject: Fix https://github.com/pyload/pyload/issues/943 --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ba718950d..809e89c02 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -33,7 +33,7 @@ def _error(self, reason, type): #@TODO: Remove in 0.4.10 def _wait(self, seconds, reconnect): if seconds: - self.setWait(seconds + 1) + self.setWait(int(seconds) + 1) if reconnect is not None: self.wantReconnect = reconnect @@ -144,7 +144,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.69" + __version__ = "0.70" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From 5a23de6f32dc99960cccd634d2fd2cc47c34be2a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 5 Dec 2014 23:08:44 +0100 Subject: Code cosmetics about checkErrors --- module/plugins/internal/SimpleHoster.py | 4 ++-- module/plugins/internal/XFSHoster.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 809e89c02..f391445fd 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -399,8 +399,8 @@ class SimpleHoster(Hoster): if hasattr(self, 'ERROR_PATTERN'): m = re.search(self.ERROR_PATTERN, self.html) if m: - e = self.info['error'] = m.group(1) - self.error(e) + errmsg = self.info['error'] = m.group(1) + self.error(errmsg) if hasattr(self, 'PREMIUM_ONLY_PATTERN'): m = re.search(self.PREMIUM_ONLY_PATTERN, self.html) diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index c3db3f335..3ae0692dc 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -256,8 +256,6 @@ class XFSHoster(SimpleHoster): else: self.info.pop('error', None) - return self.errmsg - def getPostParameters(self): if self.FORM_PATTERN or self.FORM_INPUTS_MAP: -- cgit v1.2.3 From dcf495b582440c33d878115556ceddf0f523a1ab Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 6 Dec 2014 11:02:24 +0100 Subject: [MultiHoster] Code cosmetics --- module/plugins/internal/MultiHoster.py | 45 +++++++++++++++++----------------- 1 file changed, 23 insertions(+), 22 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index c86f9b286..6ec2e4b82 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,25 +9,28 @@ from module.utils import remove_chars class MultiHoster(Hook): __name__ = "MultiHoster" __type__ = "hook" - __version__ = "0.19" + __version__ = "0.20" __description__ = """Generic MultiHoster plugin""" __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] - interval = 24 * 60 * 60 #: reload hosters daily + interval = 12 * 60 * 60 #: reload hosters every 12h - HOSTER_REPLACEMENTS = [("2shared.com", "twoshared.com"), ("4shared.com", "fourshared.com"), ("cloudnator.com", "shragle.com"), - ("ifile.it", "filecloud.io"), ("easy-share.com", "crocko.com"), ("freakshare.net", "freakshare.com"), - ("hellshare.com", "hellshare.cz"), ("share-rapid.cz", "sharerapid.com"), ("sharerapid.cz", "sharerapid.com"), - ("ul.to", "uploaded.to"), ("uploaded.net", "uploaded.to"), ("1fichier.com", "onefichier.com")] + HOSTER_REPLACEMENTS = [("1fichier.com", "onefichier.com"), ("2shared.com", "twoshared.com"), + ("4shared.com", "fourshared.com"), ("cloudnator.com", "shragle.com"), + ("easy-share.com", "crocko.com"), ("freakshare.net", "freakshare.com"), + ("hellshare.com", "hellshare.cz"), ("ifile.it", "filecloud.io"), + ("putlocker.com", "firedrive.com"), ("share-rapid.cz", "multishare.cz"), + ("sharerapid.cz", "multishare.cz"), ("ul.to", "uploaded.to"), + ("uploaded.net", "uploaded.to")] HOSTER_EXCLUDED = [] def setup(self): - self.hosters = [] - self.supported = [] + self.hosters = [] + self.supported = [] self.new_supported = [] @@ -41,7 +44,6 @@ class MultiHoster(Hook): def getHosterCached(self): if not self.hosters: - try: hosterSet = self.toHosterSet(self.getHoster()) - set(self.HOSTER_EXCLUDED) except Exception, e: @@ -110,8 +112,10 @@ class MultiHoster(Hook): """reload hoster list periodically""" self.logInfo(_("Reloading supported hoster list")) - old_supported = self.supported - self.supported, self.new_supported, self.hosters = [], [], [] + old_supported = self.supported + self.supported = [] + self.new_supported = [] + self.hosters = [] self.overridePlugins() @@ -123,11 +127,8 @@ class MultiHoster(Hook): def overridePlugins(self): - pluginMap = {} - for name in self.core.pluginManager.hosterPlugins.keys(): - pluginMap[name.lower()] = name - - accountList = [name.lower() for name, data in self.core.accountManager.accounts.iteritems() if data] + pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.keys()) + accountList = [name.lower() for name, data in self.core.accountManager.accounts.iteritems() if data] excludedList = [] for hoster in self.getHosterCached(): @@ -146,14 +147,14 @@ class MultiHoster(Hook): return module = self.core.pluginManager.getPlugin(self.__name__) - klass = getattr(module, self.__name__) + klass = getattr(module, self.__name__) # inject plugin plugin self.logDebug("Overwritten Hosters", ", ".join(sorted(self.supported))) for hoster in self.supported: dict = self.core.pluginManager.hosterPlugins[hoster] dict['new_module'] = module - dict['new_name'] = self.__name__ + dict['new_name'] = self.__name__ if excludedList: self.logInfo(_("The following hosters were not overwritten - account exists"), ", ".join(sorted(excludedList))) @@ -162,7 +163,7 @@ class MultiHoster(Hook): self.logDebug("New Hosters", ", ".join(sorted(self.new_supported))) # create new regexp - regexp = r'.*(%s).*' % "|".join([x.replace(".", "\\.") for x in self.new_supported]) + regexp = r'.*(%s).*' % "|".join([x.replace(".", "\.") for x in self.new_supported]) if hasattr(klass, "__pattern__") and isinstance(klass.__pattern__, basestring) and '://' in klass.__pattern__: regexp = r'%s|%s' % (klass.__pattern__, regexp) @@ -170,7 +171,7 @@ class MultiHoster(Hook): dict = self.core.pluginManager.hosterPlugins[self.__name__] dict['pattern'] = regexp - dict['re'] = re.compile(regexp) + dict['re'] = re.compile(regexp) def unloadHoster(self, hoster): @@ -190,9 +191,9 @@ class MultiHoster(Hook): # reset pattern klass = getattr(self.core.pluginManager.getPlugin(self.__name__), self.__name__) - dict = self.core.pluginManager.hosterPlugins[self.__name__] + dict = self.core.pluginManager.hosterPlugins[self.__name__] dict['pattern'] = getattr(klass, "__pattern__", r'^unmatchable$') - dict['re'] = re.compile(dict['pattern']) + dict['re'] = re.compile(dict['pattern']) def downloadFailed(self, pyfile): -- cgit v1.2.3 From ef0d8692b04d75f4a39a565d52c2eb381eb43055 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 6 Dec 2014 11:03:57 +0100 Subject: [SimpleHoster] Fix multihost --- module/plugins/internal/SimpleHoster.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index f391445fd..530b67692 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -144,7 +144,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.70" + __version__ = "0.71" __pattern__ = r'^unmatchable$' @@ -319,8 +319,8 @@ class SimpleHoster(Hoster): set_cookies(self.req.cj, self.COOKIES) if (self.MULTI_HOSTER - and self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] - and re.match(self.__pattern__, self.pyfile.url) is None): + and (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] + or re.match(self.__pattern__, self.pyfile.url) is None)): self.logInfo("Multi hoster detected") @@ -384,12 +384,17 @@ class SimpleHoster(Hoster): self.logDebug("Handled as free download") self.handleFree() - if self.link: - self.download(self.link, disposition=self.CONTENT_DISPOSITION) - + self.downloadLink(self.link) self.checkFile() + def downloadLink(self, link): + if not link: + return + + self.download(link, disposition=self.CONTENT_DISPOSITION) + + def checkFile(self): if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": #@TODO: Move to hoster in 0.4.10 self.fail(_("Empty file")) -- cgit v1.2.3 From 6151e81fa0b325dffda3da4228d5821e73db3ef3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 9 Dec 2014 01:19:46 +0100 Subject: Fix __version__ format in some plugins --- module/plugins/internal/AbstractExtractor.py | 2 +- module/plugins/internal/UnZip.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index 8a69ebb56..2317ad689 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -14,7 +14,7 @@ class WrongPassword(Exception): class AbtractExtractor: __name__ = "AbtractExtractor" - __version__ = "0.1" + __version__ = "0.10" __description__ = """Abtract extractor plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index e754141a1..053946dbe 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -8,7 +8,7 @@ from module.plugins.internal.AbstractExtractor import AbtractExtractor class UnZip(AbtractExtractor): __name__ = "UnZip" - __version__ = "0.1" + __version__ = "0.10" __description__ = """Zip extractor plugin""" __license__ = "GPLv3" -- cgit v1.2.3 From 58656ab0b1b7f79ee31c38b7675f0587269f014f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 10 Dec 2014 02:59:27 +0100 Subject: [CaptchaService] New service AdYouLike + AdsCaptcha detect_key fixup --- module/plugins/internal/CaptchaService.py | 203 +++++++++++++++++++++++------- 1 file changed, 161 insertions(+), 42 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index 7009e6986..965799e8e 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -4,10 +4,12 @@ import re from random import random +from module.common.json_layer import json_loads + class CaptchaService: __name__ = "CaptchaService" - __version__ = "0.15" + __version__ = "0.16" __description__ = """Base captcha service plugin""" __license__ = "GPLv3" @@ -34,7 +36,7 @@ class CaptchaService: m = re.search(self.KEY_PATTERN, html) if m: - self.key = m.group("KEY") + self.key = m.group(1).strip() self.plugin.logDebug("%s key: %s" % (self.__name__, self.key)) return self.key else: @@ -59,8 +61,8 @@ class ReCaptcha(CaptchaService): __authors__ = [("pyLoad Team", "admin@pyload.org")] - KEY_PATTERN = r'recaptcha(/api|\.net)/(challenge|noscript)\?k=(?P[\w-]+)' - KEY_AJAX_PATTERN = r'Recaptcha\.create\s*\(\s*["\'](?P[\w-]+)' + KEY_PATTERN = r'recaptcha(?:/api|\.net)/(?:challenge|noscript)\?k=([\w-]+)' + KEY_AJAX_PATTERN = r'Recaptcha\.create\s*\(\s*["\']([\w-]+)' def detect_key(self, html=None): @@ -74,7 +76,7 @@ class ReCaptcha(CaptchaService): m = re.search(self.KEY_PATTERN, html) or re.search(self.KEY_AJAX_PATTERN, html) if m: - self.key = m.group("KEY") + self.key = m.group(1).strip() self.plugin.logDebug("ReCaptcha key: %s" % self.key) return self.key else: @@ -91,36 +93,43 @@ class ReCaptcha(CaptchaService): self.plugin.fail(errmsg) raise TypeError(errmsg) - js = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}) + html = self.plugin.req.load("http://www.google.com/recaptcha/api/challenge", get={'k': key}) try: - challenge = re.search("challenge : '(.+?)',", js).group(1) - server = re.search("server : '(.+?)',", js).group(1) + challenge = re.search("challenge : '(.+?)',", html).group(1) + server = re.search("server : '(.+?)',", html).group(1) except: - self.plugin.error("ReCaptcha challenge pattern not found") - - result = self.result(server, challenge) + errmsg = _("ReCaptcha challenge pattern not found") + self.plugin.fail(errmsg) + raise ValueError(errmsg) - self.plugin.logDebug("ReCaptcha result: %s" % result, "challenge: %s" % challenge) + self.plugin.logDebug("ReCaptcha challenge: %s" % challenge) - return challenge, result + return challenge, self.result(server, challenge) def result(self, server, challenge): - return self.plugin.decryptCaptcha("%simage" % server, get={'c': challenge}, - cookies=True, forceUser=True, imgtype="jpg") + result = self.plugin.decryptCaptcha("%simage" % server, + get={'c': challenge}, + cookies=True, + forceUser=True, + imgtype="jpg") + + self.plugin.logDebug("ReCaptcha result: %s" % result) + + return result class AdsCaptcha(CaptchaService): __name__ = "AdsCaptcha" - __version__ = "0.05" + __version__ = "0.06" __description__ = """AdsCaptcha captcha service plugin""" __license__ = "GPLv3" __authors__ = [("pyLoad Team", "admin@pyload.org")] - ID_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*CaptchaId=(?P\d+)' - KEY_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*PublicKey=(?P[\w-]+)' + CAPTCHAID_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*CaptchaId=(\d+)' + PUBLICKEY_PATTERN = r'api\.adscaptcha\.com/Get\.aspx\?[^"\']*PublicKey=([\w-]+)' def detect_key(self, html=None): @@ -132,18 +141,18 @@ class AdsCaptcha(CaptchaService): self.plugin.fail(errmsg) raise TypeError(errmsg) - m = re.search(self.ID_PATTERN, html) - n = re.search(self.KEY_PATTERN, html) + m = re.search(self.PUBLICKEY_PATTERN, html) + n = re.search(self.CAPTCHAID_PATTERN, html) if m and n: - self.key = (m.group("ID"), m.group("KEY")) - self.plugin.logDebug("AdsCaptcha id|key: %s | %s" % self.key) + self.key = (m.group(1).strip(), n.group(1).strip()) #: key is the tuple(PublicKey, CaptchaId) + self.plugin.logDebug("AdsCaptcha key|id: %s | %s" % self.key) return self.key else: - self.plugin.logDebug("AdsCaptcha id or key not found") + self.plugin.logDebug("AdsCaptcha key or id not found") return None - def challenge(self, key=None): #: key is a tuple(CaptchaId, PublicKey) + def challenge(self, key=None): if not key: if self.detect_key(): key = self.key @@ -152,25 +161,31 @@ class AdsCaptcha(CaptchaService): self.plugin.fail(errmsg) raise TypeError(errmsg) - CaptchaId, PublicKey = key + PublicKey, CaptchaId = key - js = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}) + html = self.plugin.req.load("http://api.adscaptcha.com/Get.aspx", get={'CaptchaId': CaptchaId, 'PublicKey': PublicKey}) try: - challenge = re.search("challenge: '(.+?)',", js).group(1) - server = re.search("server: '(.+?)',", js).group(1) + challenge = re.search("challenge: '(.+?)',", html).group(1) + server = re.search("server: '(.+?)',", html).group(1) except: - self.plugin.error("AdsCaptcha challenge pattern not found") - - result = self.result(server, challenge) + errmsg = _("AdsCaptcha challenge pattern not found") + self.plugin.fail(errmsg) + raise ValueError(errmsg) - self.plugin.logDebug("AdsCaptcha result: %s" % result, "challenge: %s" % challenge) + self.plugin.logDebug("AdsCaptcha challenge: %s" % challenge) - return challenge, result + return challenge, self.result(server, challenge) def result(self, server, challenge): - return self.plugin.decryptCaptcha("%sChallenge.aspx" % server, get={'cid': challenge, 'dummy': random()}, - cookies=True, imgtype="jpg") + result = self.plugin.decryptCaptcha("%sChallenge.aspx" % server, + get={'cid': challenge, 'dummy': random()}, + cookies=True, + imgtype="jpg") + + self.plugin.logDebug("AdsCaptcha result: %s" % result) + + return result class SolveMedia(CaptchaService): @@ -182,7 +197,7 @@ class SolveMedia(CaptchaService): __authors__ = [("pyLoad Team", "admin@pyload.org")] - KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(no)?script\?k=(?P.+?)["\']' + KEY_PATTERN = r'api\.solvemedia\.com/papi/challenge\.(?:no)?script\?k=(.+?)["\']' def challenge(self, key=None): @@ -198,16 +213,120 @@ class SolveMedia(CaptchaService): try: challenge = re.search(r'', html).group(1) - server = "http://api.solvemedia.com/papi/media" + server = "http://api.solvemedia.com/papi/media" except: - self.plugin.error("SolveMedia challenge pattern not found") + errmsg = _("SolveMedia challenge pattern not found") + self.plugin.fail(errmsg) + raise ValueError(errmsg) + + self.plugin.logDebug("SolveMedia challenge: %s" % challenge) + + return challenge, self.result(server, challenge) + + + def result(self, server, challenge): + result = self.plugin.decryptCaptcha(server, get={'c': challenge}, imgtype="gif") + + self.plugin.logDebug("SolveMedia result: %s" % result) + + return result + + +class AdYouLike(CaptchaService): + __name__ = "AdYouLike" + __version__ = "0.02" + + __description__ = """AdYouLike captcha service plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + AYL_PATTERN = r'Adyoulike\.create\s*\((.+?)\)' + CALLBACK_PATTERN = r'(Adyoulike\.g\._jsonp_\d+)' + + + def detect_key(self, html=None): + if not html: + if hasattr(self.plugin, "html") and self.plugin.html: + html = self.plugin.html + else: + errmsg = _("AdYouLike html not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) - result = self.result(server, challenge) + m = re.search(self.AYL_PATTERN, html) + n = re.search(self.CALLBACK_PATTERN, html) + if m and n: + self.key = (m.group(1).strip(), n.group(1).strip()) + self.plugin.logDebug("AdYouLike ayl|callback: %s | %s" % self.key) + return self.key #: key is the tuple(ayl, callback) + else: + self.plugin.logDebug("AdYouLike ayl or callback not found") + return None - self.plugin.logDebug("SolveMedia result: %s" % result, "challenge: %s" % challenge) - return challenge, result + def challenge(self, key=None): + if not key: + if self.detect_key(): + key = self.key + else: + errmsg = _("AdYouLike key not found") + self.plugin.fail(errmsg) + raise TypeError(errmsg) + + ayl, callback = key + + # {"adyoulike":{"key":"P~zQ~O0zV0WTiAzC-iw0navWQpCLoYEP"}, + # "all":{"element_id":"ayl_private_cap_92300","lang":"fr","env":"prod"}} + ayl = json_loads(ayl) + + html = self.plugin.req.load("http://api-ayl.appspot.com/challenge", + get={'key' : ayl['adyoulike']['key'], + 'env' : ayl['all']['env'], + 'callback': callback}) + try: + challenge = json_loads(re.search(callback + r'\s*\((.+?)\)', html).group(1)) + except: + errmsg = _("AdYouLike challenge pattern not found") + self.plugin.fail(errmsg) + raise ValueError(errmsg) + + self.plugin.logDebug("AdYouLike challenge: %s" % challenge) + + return self.result(ayl, challenge) def result(self, server, challenge): - return self.plugin.decryptCaptcha(server, get={'c': challenge}, imgtype="gif") + # Adyoulike.g._jsonp_5579316662423138 + # ({"translations":{"fr":{"instructions_visual":"Recopiez « Soonnight » ci-dessous :"}}, + # "site_under":true,"clickable":true,"pixels":{"VIDEO_050":[],"DISPLAY":[],"VIDEO_000":[],"VIDEO_100":[], + # "VIDEO_025":[],"VIDEO_075":[]},"medium_type":"image/adyoulike", + # "iframes":{"big":""},"shares":{},"id":256, + # "token":"e6QuI4aRSnbIZJg02IsV6cp4JQ9~MjA1","formats":{"small":{"y":300,"x":0,"w":300,"h":60}, + # "big":{"y":0,"x":0,"w":300,"h":250},"hover":{"y":440,"x":0,"w":300,"h":60}}, + # "tid":"SqwuAdxT1EZoi4B5q0T63LN2AkiCJBg5"}) + + if isinstance(server, basestring): + server = json_loads(server) + + if isinstance(challenge, basestring): + challenge = json_loads(challenge) + + try: + instructions_visual = challenge['translations'][server['all']['lang']]['instructions_visual'] + result = re.search(u'«(.+?)»', instructions_visual).group(1).strip() + except: + errmsg = _("AdYouLike result not found") + self.plugin.fail(errmsg) + raise ValueError(errmsg) + + result = {'_ayl_captcha_engine' : "adyoulike", + '_ayl_env' : server['all']['env'], + '_ayl_tid' : challenge['tid'], + '_ayl_token_challenge': challenge['token'], + '_ayl_response' : response} + + self.plugin.logDebug("AdYouLike result: %s" % result) + + return result -- cgit v1.2.3 From 1fe525b92affd6d3829af219a9b1b5eeda136ca2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 12 Dec 2014 20:31:37 +0100 Subject: [SimpleHoster] Remove CONTENT_DISPOSITION + Rename parseInfo to parseInfos --- module/plugins/internal/SimpleHoster.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 530b67692..ddaea020a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -104,9 +104,9 @@ def parseFileInfo(plugin, url="", html=""): #@TODO: Remove in 0.4.10 -#@NOTE: Every plugin must have own parseInfo classmethod to work with 0.4.10 +#@NOTE: Every plugin must have own parseInfos classmethod to work with 0.4.10 def create_getInfo(plugin): - return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfo(urls)] + return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfos(urls)] def timestamp(): @@ -144,7 +144,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.71" + __version__ = "0.72" __pattern__ = r'^unmatchable$' @@ -206,11 +206,10 @@ class SimpleHoster(Hoster): FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account CHECK_DIRECT_LINK = None #: Set to True to check for direct link, set to None to do it only if self.account is True MULTI_HOSTER = False #: Set to True to leech other hoster link (according its multihoster hook if available) - CONTENT_DISPOSITION = False #: Set to True to replace file name with content-disposition value from http header @classmethod - def parseInfo(cls, urls): + def parseInfos(cls, urls): for url in urls: url = replace_patterns(url, cls.FILE_URL_REPLACEMENTS if hasattr(cls, "FILE_URL_REPLACEMENTS") else cls.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 yield cls.getInfo(url) @@ -392,7 +391,7 @@ class SimpleHoster(Hoster): if not link: return - self.download(link, disposition=self.CONTENT_DISPOSITION) + self.download(link, disposition=True) def checkFile(self): -- cgit v1.2.3 From 0f530be2642c63759cb6156866f1d4ab119760da Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 12 Dec 2014 20:33:34 +0100 Subject: Fix missing create_getInfo --- module/plugins/internal/XFSCrypter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 62fd8c017..4b57dab90 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.SimpleCrypter import SimpleCrypter +from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.04" + __version__ = "0.05" __pattern__ = r'^unmatchable$' -- cgit v1.2.3 From c9b4f3b9a78904d34d7e58593eacdc26b93c2562 Mon Sep 17 00:00:00 2001 From: philou75 Date: Fri, 12 Dec 2014 22:49:23 +0100 Subject: Make the waiting time work again on OneFichier.com --- module/plugins/internal/SimpleHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index f391445fd..47b713173 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -144,7 +144,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.70" + __version__ = "0.71" __pattern__ = r'^unmatchable$' @@ -412,7 +412,7 @@ class SimpleHoster(Hoster): m = re.search(self.WAIT_PATTERN, self.html) if m: wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in - re.findall(r'(\d+)\s*(hr|hour|min|sec)', m, re.I)]) + re.findall(r'(\d+)\s*(hr|hour|min|sec)', m.group(0), re.I)]) self.wait(wait_time, False) return -- cgit v1.2.3 From 3d27f5ccee412d38102873a5b02e3f236375eb97 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 16 Dec 2014 03:44:15 +0100 Subject: Update plugins (2) --- module/plugins/internal/MultiHoster.py | 50 +++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 22 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 6ec2e4b82..6d10d17d9 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.utils import remove_chars class MultiHoster(Hook): __name__ = "MultiHoster" __type__ = "hook" - __version__ = "0.20" + __version__ = "0.21" __description__ = """Generic MultiHoster plugin""" __license__ = "GPLv3" @@ -18,12 +18,18 @@ class MultiHoster(Hook): interval = 12 * 60 * 60 #: reload hosters every 12h - HOSTER_REPLACEMENTS = [("1fichier.com", "onefichier.com"), ("2shared.com", "twoshared.com"), - ("4shared.com", "fourshared.com"), ("cloudnator.com", "shragle.com"), - ("easy-share.com", "crocko.com"), ("freakshare.net", "freakshare.com"), - ("hellshare.com", "hellshare.cz"), ("ifile.it", "filecloud.io"), - ("putlocker.com", "firedrive.com"), ("share-rapid.cz", "multishare.cz"), - ("sharerapid.cz", "multishare.cz"), ("ul.to", "uploaded.to"), + HOSTER_REPLACEMENTS = [("1fichier.com", "onefichier.com"), + ("2shared.com", "twoshared.com"), + ("4shared.com", "fourshared.com"), + ("cloudnator.com", "shragle.com"), + ("easy-share.com", "crocko.com"), + ("freakshare.net", "freakshare.com"), + ("hellshare.com", "hellshare.cz"), + ("ifile.it", "filecloud.io"), + ("putlocker.com", "firedrive.com"), + ("share-rapid.cz", "multishare.cz"), + ("sharerapid.cz", "multishare.cz"), + ("ul.to", "uploaded.to"), ("uploaded.net", "uploaded.to")] HOSTER_EXCLUDED = [] @@ -152,9 +158,9 @@ class MultiHoster(Hook): # inject plugin plugin self.logDebug("Overwritten Hosters", ", ".join(sorted(self.supported))) for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - dict['new_module'] = module - dict['new_name'] = self.__name__ + hdict = self.core.pluginManager.hosterPlugins[hoster] + hdict['new_module'] = module + hdict['new_name'] = self.__name__ if excludedList: self.logInfo(_("The following hosters were not overwritten - account exists"), ", ".join(sorted(excludedList))) @@ -169,19 +175,19 @@ class MultiHoster(Hook): self.logDebug("Regexp", regexp) - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict['pattern'] = regexp - dict['re'] = re.compile(regexp) + hdict = self.core.pluginManager.hosterPlugins[self.__name__] + hdict['pattern'] = regexp + hdict['re'] = re.compile(regexp) def unloadHoster(self, hoster): - dict = self.core.pluginManager.hosterPlugins[hoster] - if "module" in dict: - del dict['module'] + hdict = self.core.pluginManager.hosterPlugins[hoster] + if "module" in hdict: + del hdict['module'] - if "new_module" in dict: - del dict['new_module'] - del dict['new_name'] + if "new_module" in hdict: + del hdict['new_module'] + del hdict['new_name'] def unload(self): @@ -191,9 +197,9 @@ class MultiHoster(Hook): # reset pattern klass = getattr(self.core.pluginManager.getPlugin(self.__name__), self.__name__) - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict['pattern'] = getattr(klass, "__pattern__", r'^unmatchable$') - dict['re'] = re.compile(dict['pattern']) + hdict = self.core.pluginManager.hosterPlugins[self.__name__] + hdict['pattern'] = getattr(klass, "__pattern__", r'^unmatchable$') + hdict['re'] = re.compile(hdict['pattern']) def downloadFailed(self, pyfile): -- cgit v1.2.3 From f633832bbfce04f4b1b83e59c02701483ef03525 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 16 Dec 2014 23:01:43 +0100 Subject: [XFSHoster] Improve SIZE_PATTERN --- module/plugins/internal/XFSHoster.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 3ae0692dc..bb4baa274 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.27" + __version__ = "0.28" __pattern__ = r'^unmatchable$' @@ -36,7 +36,7 @@ class XFSHoster(SimpleHoster): MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... NAME_PATTERN = r'(>Filename:|name="fname" value="|)(?P.+?)(\s*<|")' - SIZE_PATTERN = r'(>Size:|>File:.*>|)(?P[\d.,]+)\s*(?P[\w^_]+)' + SIZE_PATTERN = r'(>Size:|>File:.*>|\s*\(|)(?P[\d.,]+)\s*(?P[\w^_]+)' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' @@ -199,13 +199,7 @@ class XFSHoster(SimpleHoster): def checkErrors(self): - m = re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if m: - self.info['error'] = "premium-only" - return - m = re.search(self.ERROR_PATTERN, self.html) - if m is None: self.errmsg = None else: -- cgit v1.2.3 From b18bf426955d85d6cb3d046aa8b074dfe56340c1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 16 Dec 2014 23:02:20 +0100 Subject: [UnZip] Password support --- module/plugins/internal/UnZip.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 053946dbe..52e279ccf 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -3,12 +3,12 @@ import sys import zipfile -from module.plugins.internal.AbstractExtractor import AbtractExtractor +from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError class UnZip(AbtractExtractor): __name__ = "UnZip" - __version__ = "0.10" + __version__ = "0.11" __description__ = """Zip extractor plugin""" __license__ = "GPLv3" @@ -32,9 +32,19 @@ class UnZip(AbtractExtractor): def extract(self, progress, password=None): - z = zipfile.ZipFile(self.file) - self.files = z.namelist() - z.extractall(self.out) + try: + z = zipfile.ZipFile(self.file) + self.files = z.namelist() + z.extractall(self.out, pwd=password) + + except (BadZipfile, LargeZipFile), e: + raise ArchiveError(e) + + except RuntimeError, e: + if e is "Bad password for file": + raise WrongPassword + else: + raise ArchiveError(e) def getDeleteFiles(self): -- cgit v1.2.3 From a51d35534fc38650dd7714572f6874ab262bd119 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 16 Dec 2014 23:03:05 +0100 Subject: [SimpleHoster] Improve and fix --- module/plugins/internal/SimpleHoster.py | 58 +++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 24 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 992454451..86d9b4c9a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -99,14 +99,24 @@ def parseHtmlForm(attr_str, html, input_names={}): #: Deprecated def parseFileInfo(plugin, url="", html=""): - info = plugin.getInfo(url, html) - return info['name'], info['size'], info['status'], info['url'] + if hasattr(plugin, "getInfo"): + info = plugin.getInfo(url, html) + res = info['name'], info['size'], info['status'], info['url'] + else: + res = urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 0, 3, url + + return res #@TODO: Remove in 0.4.10 #@NOTE: Every plugin must have own parseInfos classmethod to work with 0.4.10 def create_getInfo(plugin): - return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfos(urls)] + if hasattr(plugin, "parseInfos"): + fn = lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfos(urls)] + else: + fn = lambda urls: [parseFileInfo(url) for url in urls] + + return fn def timestamp(): @@ -144,7 +154,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.73" + __version__ = "0.74" __pattern__ = r'^unmatchable$' @@ -260,18 +270,18 @@ class SimpleHoster(Hoster): try: info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here, please save api stuff to info['api'] except: - pass + info['pattern'] = {} for pattern in ("FILE_INFO_PATTERN", "INFO_PATTERN", "FILE_NAME_PATTERN", "NAME_PATTERN", "FILE_SIZE_PATTERN", "SIZE_PATTERN", "HASHSUM_PATTERN"): #@TODO: Remove old patterns starting with "FILE_" in 0.4.10 try: - attr = getattr(cls, pattern) - dict = re.search(attr, html).groupdict() + attr = getattr(cls, pattern) + pdict = re.search(attr, html).groupdict() - if all(True for k in dict if k not in info['pattern']): - info['pattern'].update(dict) + if all(True for k in pdict if k not in info['pattern']): + info['pattern'].update(pdict) except AttributeError: continue @@ -279,6 +289,9 @@ class SimpleHoster(Hoster): else: online = True + if not info['pattern']: + info.pop('pattern', None) + if online: info['status'] = 2 @@ -360,18 +373,21 @@ class SimpleHoster(Hoster): if self.html is None: self.fail(_("No html retrieved")) - self.checkErrors() - - premium_only = 'error' in self.info and self.info['error'] == "premium-only" - - self._updateInfo(self.getInfo(pyfile.url, self.html)) + self.updateInfo(self.getInfo(pyfile.url, self.html)) self.checkNameSize() + if hasattr(self, 'PREMIUM_ONLY_PATTERN'): + premium_only = re.search(self.PREMIUM_ONLY_PATTERN, self.html) + else: + premium_only = False + #: Usually premium only pages doesn't show any file information if not premium_only: self.checkStatus() + self.checkErrors() + if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): self.logDebug("Handled as premium download") self.handlePremium() @@ -406,18 +422,12 @@ class SimpleHoster(Hoster): errmsg = self.info['error'] = m.group(1) self.error(errmsg) - if hasattr(self, 'PREMIUM_ONLY_PATTERN'): - m = re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if m: - self.info['error'] = "premium-only" - return - if hasattr(self, 'WAIT_PATTERN'): m = re.search(self.WAIT_PATTERN, self.html) if m: wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in re.findall(r'(\d+)\s*(hr|hour|min|sec)', m.group(0), re.I)]) - self.wait(wait_time, True if wait_time > 300 else False) + self.wait(wait_time, wait_time > 300) return self.info.pop('error', None) @@ -460,7 +470,7 @@ class SimpleHoster(Hoster): def checkInfo(self): self.checkErrors() - self._updateInfo(self.getInfo(self.pyfile.url, self.html or "")) + self.updateInfo(self.getInfo(self.pyfile.url, self.html or "")) self.checkNameSize() self.checkStatus() @@ -473,7 +483,7 @@ class SimpleHoster(Hoster): return self.info - def _updateInfo(self, info): + def updateInfo(self, info): self.logDebug(_("File info (before update): %s") % self.info) self.info.update(info) self.logDebug(_("File info (after update): %s") % self.info) @@ -487,7 +497,7 @@ class SimpleHoster(Hoster): self.link = link - self._updateInfo(self.getInfo(self.pyfile.url)) + self.updateInfo(self.getInfo(self.pyfile.url)) self.checkNameSize() else: self.logDebug(_("Direct download link not found")) -- cgit v1.2.3 From 8bb189cf5495b293cf574e5049c34a64ae36ffe1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 16 Dec 2014 23:04:15 +0100 Subject: Spare code cosmetics --- module/plugins/internal/UnRar.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index c15a4c96e..716a62613 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(AbtractExtractor): __name__ = "UnRar" - __version__ = "0.19" + __version__ = "0.20" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" @@ -32,7 +32,7 @@ class UnRar(AbtractExtractor): CMD = "unrar" # there are some more uncovered rar formats - re_version = re.compile(r'UNRAR ([\w .]+?) freeware') + re_version = re.compile(r'UNRAR ([\w .]+?)') re_splitfile = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) re_partfiles = re.compile(r'.*\.(rar|r\d+)', re.I) re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+') @@ -91,7 +91,7 @@ class UnRar(AbtractExtractor): out, err = p.communicate() if self.re_wrongpwd.search(err): self.passwordProtected = True - self.headerProtected = True + self.headerProtected = True return True # output only used to check if passworded files are present @@ -124,7 +124,7 @@ class UnRar(AbtractExtractor): return True - def extract(self, progress, password=None): + def extract(self, progress, password=""): command = "x" if self.fullpath else "e" p = self.call_unrar(command, self.file, self.out, password=password) @@ -156,6 +156,7 @@ class UnRar(AbtractExtractor): raise CRCError elif "CRC failed" in err: raise WrongPassword + if err.strip(): #: raise error if anything is on stderr raise ArchiveError(err.strip()) if p.returncode: -- cgit v1.2.3 From 69616099c133fdc783abad44c2a0916acba4b6d1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 17 Dec 2014 22:38:24 +0100 Subject: [SimpleHoster] Fix filesize recognition --- module/plugins/internal/SimpleHoster.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 86d9b4c9a..ad140423c 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -154,7 +154,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.74" + __version__ = "0.75" __pattern__ = r'^unmatchable$' @@ -300,7 +300,7 @@ class SimpleHoster(Hoster): cls.FILE_NAME_REPLACEMENTS if hasattr(cls, "FILE_NAME_REPLACEMENTS") else cls.NAME_REPLACEMENTS) #@TODO: Remove FILE_NAME_REPLACEMENTS check in 0.4.10 if 'S' in info['pattern']: - size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info else info['pattern']['S'], + size = replace_patterns(info['pattern']['S'] + info['pattern']['U'] if 'U' in info['pattern'] else info['pattern']['S'], cls.FILE_SIZE_REPLACEMENTS if hasattr(cls, "FILE_SIZE_REPLACEMENTS") else cls.SIZE_REPLACEMENTS) #@TODO: Remove FILE_SIZE_REPLACEMENTS check in 0.4.10 info['size'] = parseFileSize(size) @@ -361,6 +361,7 @@ class SimpleHoster(Hoster): if self.multihost: self.logDebug("Looking for leeched download link...") + self.logDebug("File url: %s" % self.pyfile.url) self.handleMulti() elif self.directDL: @@ -404,10 +405,8 @@ class SimpleHoster(Hoster): def downloadLink(self, link): - if not link: - return - - self.download(link, disposition=True) + if link and isinstance(link, basestring): + self.download(link, disposition=True) def checkFile(self): -- cgit v1.2.3 From 0860e09f5ff16ee3f097f6f9d444f277a38abd72 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 17 Dec 2014 23:03:46 +0100 Subject: Extend SimpleHoster in multi-hoster plugins (3) --- module/plugins/internal/MultiHoster.py | 41 ++++++++++++++++++--------------- module/plugins/internal/SimpleHoster.py | 2 +- 2 files changed, 24 insertions(+), 19 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 6d10d17d9..0bcb79817 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.utils import remove_chars class MultiHoster(Hook): __name__ = "MultiHoster" __type__ = "hook" - __version__ = "0.21" + __version__ = "0.22" __description__ = """Generic MultiHoster plugin""" __license__ = "GPLv3" @@ -18,19 +18,22 @@ class MultiHoster(Hook): interval = 12 * 60 * 60 #: reload hosters every 12h - HOSTER_REPLACEMENTS = [("1fichier.com", "onefichier.com"), - ("2shared.com", "twoshared.com"), - ("4shared.com", "fourshared.com"), - ("cloudnator.com", "shragle.com"), - ("easy-share.com", "crocko.com"), - ("freakshare.net", "freakshare.com"), - ("hellshare.com", "hellshare.cz"), - ("ifile.it", "filecloud.io"), - ("putlocker.com", "firedrive.com"), - ("share-rapid.cz", "multishare.cz"), - ("sharerapid.cz", "multishare.cz"), - ("ul.to", "uploaded.to"), - ("uploaded.net", "uploaded.to")] + HOSTER_REPLACEMENTS = [("1fichier.com" , "onefichier.com"), + ("2shared.com" , "twoshared.com" ), + ("4shared.com" , "fourshared.com"), + ("cloudnator.com" , "shragle.com" ), + ("easy-share.com" , "crocko.com" ), + ("fileparadox.com", "fileparadox.in"), + ("freakshare.net" , "freakshare.com"), + ("hellshare.com" , "hellshare.cz" ), + ("ifile.it" , "filecloud.io" ), + ("nowdownload.ch" , "nowdownload.sx"), + ("nowvideo.co" , "nowvideo.sx" ), + ("putlocker.com" , "firedrive.com" ), + ("share-rapid.cz" , "multishare.cz" ), + ("sharerapid.cz" , "multishare.cz" ), + ("ul.to" , "uploaded.to" ), + ("uploaded.net" , "uploaded.to" )] HOSTER_EXCLUDED = [] @@ -133,12 +136,12 @@ class MultiHoster(Hook): def overridePlugins(self): - pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.keys()) + pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys()) accountList = [name.lower() for name, data in self.core.accountManager.accounts.iteritems() if data] excludedList = [] for hoster in self.getHosterCached(): - name = remove_chars(hoster.lower(), "-.") + name = remove_chars(hoster, "-.") if name in accountList: excludedList.append(hoster) @@ -166,10 +169,12 @@ class MultiHoster(Hook): self.logInfo(_("The following hosters were not overwritten - account exists"), ", ".join(sorted(excludedList))) if self.new_supported: - self.logDebug("New Hosters", ", ".join(sorted(self.new_supported))) + hosters = sorted(self.new_supported) + + self.logDebug("New Hosters", ", ".join(hosters)) # create new regexp - regexp = r'.*(%s).*' % "|".join([x.replace(".", "\.") for x in self.new_supported]) + regexp = r'.*(%s).*' % "|".join([x.replace(".", "\.") for x in hosters]) if hasattr(klass, "__pattern__") and isinstance(klass.__pattern__, basestring) and '://' in klass.__pattern__: regexp = r'%s|%s' % (klass.__pattern__, regexp) diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ad140423c..2ff60d6ea 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -361,7 +361,7 @@ class SimpleHoster(Hoster): if self.multihost: self.logDebug("Looking for leeched download link...") - self.logDebug("File url: %s" % self.pyfile.url) + self.logDebug("File url: %s" % pyfile.url) self.handleMulti() elif self.directDL: -- cgit v1.2.3 From ad2808a5eb6903875036c00b5195e8b7651379f0 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 17 Dec 2014 23:48:23 +0100 Subject: [XFSHoster] Improve NAME_PATTERN & SIZE_PATTERN --- module/plugins/internal/XFSHoster.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index bb4baa274..622f34ea9 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.28" + __version__ = "0.29" __pattern__ = r'^unmatchable$' @@ -35,8 +35,8 @@ class XFSHoster(SimpleHoster): CHECK_DIRECT_LINK = None MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... - NAME_PATTERN = r'(>Filename:|name="fname" value="|)(?P.+?)(\s*<|")' - SIZE_PATTERN = r'(>Size:|>File:.*>|\s*\(|)(?P[\d.,]+)\s*(?P[\w^_]+)' + NAME_PATTERN = r'(Filename[ ]*:[ ]*()?|name="fname"[ ]+value="|<[\w^_]+ class="(file)?name">)\s*(?P.+?)(\s*<|")' + SIZE_PATTERN = r'(Size[ ]*:[ ]*()?|File:.*>|\s*\(|<[\w^_]+ class="size">)\s*(?P[\d.,]+)\s*(?P[\w^_]+)' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' -- cgit v1.2.3 From 4e9c8f7ab1269966a9eac9e1b6363f5458f9f970 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 18 Dec 2014 16:07:21 +0100 Subject: Update checkFile routine in some hoster plugins --- module/plugins/internal/MultiHoster.py | 2 +- module/plugins/internal/SimpleHoster.py | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 0bcb79817..68550a4dd 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -33,7 +33,7 @@ class MultiHoster(Hook): ("share-rapid.cz" , "multishare.cz" ), ("sharerapid.cz" , "multishare.cz" ), ("ul.to" , "uploaded.to" ), - ("uploaded.net" , "uploaded.to" )] + ("uploaded.net" , "uploaded.to" )] HOSTER_EXCLUDED = [] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 2ff60d6ea..949a6c66a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -2,6 +2,7 @@ import re +from os.path import exists from time import time from urllib import unquote from urlparse import urljoin, urlparse @@ -11,7 +12,7 @@ from module.network.CookieJar import CookieJar from module.network.RequestFactory import getURL from module.plugins.Hoster import Hoster from module.plugins.Plugin import Fail -from module.utils import fixup, parseFileSize +from module.utils import fixup, fs_encode, parseFileSize #@TODO: Adapt and move to PyFile in 0.4.10 @@ -154,7 +155,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.75" + __version__ = "0.76" __pattern__ = r'^unmatchable$' @@ -410,8 +411,19 @@ class SimpleHoster(Hoster): def checkFile(self): - if self.checkDownload({'empty': re.compile(r"^$")}) is "empty": #@TODO: Move to hoster in 0.4.10 - self.fail(_("Empty file")) + if not self.lastDownload or not exists(fs_encode(self.lastDownload)): + self.fail(_("No file downloaded")) + + else: + rules = {'empty file': re.compile(r"^$")} + + if hasattr(self, 'ERROR_PATTERN'): + rules['error'] = re.compile(self.ERROR_PATTERN) + + check = self.checkDownload(rules) + if check: #@TODO: Move to hoster in 0.4.10 + errmsg = check.strip().capitalize() + (" | " + self.lastCheck.strip() if self.lastCheck else "") + self.retry(10, 60, errmsg) def checkErrors(self): -- cgit v1.2.3 From da9e6c949243613f4d5e100cac6ff192449b4718 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 18 Dec 2014 18:45:17 +0100 Subject: Update extractor plugins --- module/plugins/internal/AbstractExtractor.py | 29 +++++++-------- module/plugins/internal/UnRar.py | 54 +++++++++++++--------------- module/plugins/internal/UnZip.py | 15 ++++---- 3 files changed, 47 insertions(+), 51 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index 2317ad689..c8a73c861 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -14,23 +14,24 @@ class WrongPassword(Exception): class AbtractExtractor: __name__ = "AbtractExtractor" - __version__ = "0.10" + __version__ = "0.11" __description__ = """Abtract extractor plugin""" __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] + __authors__ = [("RaNaN", "ranan@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] - @staticmethod - def checkDeps(): + @classmethod + def checkDeps(cls): """ Check if system statisfy dependencies :return: boolean """ return True - @staticmethod - def getTargets(files_ids): + @classmethod + def getTargets(cls, files_ids): """ Filter suited targets from list of filename id tuple list :param files_ids: List of filepathes :return: List of targets, id tuple list @@ -48,14 +49,14 @@ class AbtractExtractor: :param overwrite: Overwrite existing archives :param renice: Renice value """ - self.m = m - self.file = file - self.out = out - self.fullpath = fullpath - self.overwrite = overwrite + self.m = m + self.file = file + self.out = out + self.fullpath = fullpath + self.overwrite = overwrite self.excludefiles = excludefiles - self.renice = renice - self.files = [] #: Store extracted files here + self.renice = renice + self.files = [] #: Store extracted files here def init(self): @@ -83,7 +84,7 @@ class AbtractExtractor: return True - def extract(self, progress, password=None): + def extract(self, progress, password=""): """Extract the archive. Raise specific errors in case of failure. :param progress: Progress function, call this to update status diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 716a62613..4bbd2042c 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -22,53 +22,52 @@ def renice(pid, value): class UnRar(AbtractExtractor): __name__ = "UnRar" - __version__ = "0.20" + __version__ = "0.21" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org")] + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] CMD = "unrar" - # there are some more uncovered rar formats - re_version = re.compile(r'UNRAR ([\w .]+?)') + #@NOTE: there are some more uncovered rar formats re_splitfile = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) re_partfiles = re.compile(r'.*\.(rar|r\d+)', re.I) - re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+') - re_filelist5 = re.compile(r'(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') + re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') re_wrongpwd = re.compile(r'(Corrupt file or wrong password|password incorrect)', re.I) - @staticmethod - def checkDeps(): + @classmethod + def checkDeps(cls): if os.name == "nt": - UnRar.CMD = join(pypath, "UnRAR.exe") - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) + cls.CMD = join(pypath, "UnRAR.exe") + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) p.communicate() else: try: - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) p.communicate() - except OSError: + except OSError: # fallback to rar - UnRar.CMD = "rar" - p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) + cls.CMD = "rar" + p = Popen([cls.CMD], stdout=PIPE, stderr=PIPE) p.communicate() return True - @staticmethod - def getTargets(files_ids): + @classmethod + def getTargets(cls, files_ids): result = [] for file, id in files_ids: if not file.endswith(".rar"): continue - match = UnRar.re_splitfile.findall(file) + match = cls.re_splitfile.findall(file) if match: # only add first parts if int(match[0][1]) == 1: @@ -81,9 +80,8 @@ class UnRar(AbtractExtractor): def init(self): self.passwordProtected = False - self.headerProtected = False #: list files will not work without password - self.smallestFile = None #: small file to test passwords - self.password = "" #: save the correct password + self.headerProtected = False #: list files will not work without password + self.password = "" #: save the correct password def checkArchive(self): @@ -95,16 +93,10 @@ class UnRar(AbtractExtractor): return True # output only used to check if passworded files are present - if self.re_version.search(out): - for attr, size, name in self.re_filelist5.findall(out): - if attr.startswith("*"): - self.passwordProtected = True - return True - else: - for name, size, packed in self.re_filelist.findall(out): - if name.startswith("*"): - self.passwordProtected = True - return True + for attr in self.re_filelist.findall(out): + if attr[0].startswith("*"): + self.passwordProtected = True + return True self.listContent() if not self.files: @@ -131,6 +123,7 @@ class UnRar(AbtractExtractor): renice(p.pid, self.renice) progress(0) + progressstring = "" while True: c = p.stdout.read(1) @@ -147,6 +140,7 @@ class UnRar(AbtractExtractor): # add digit to progressstring else: progressstring = progressstring + c + progress(100) # retrieve stderr diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 52e279ccf..81c298784 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -8,20 +8,21 @@ from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPas class UnZip(AbtractExtractor): __name__ = "UnZip" - __version__ = "0.11" + __version__ = "0.12" __description__ = """Zip extractor plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org")] + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] - @staticmethod - def checkDeps(): + @classmethod + def checkDeps(cls): return sys.version_info[:2] >= (2, 6) - @staticmethod - def getTargets(files_ids): + @classmethod + def getTargets(cls, files_ids): result = [] for file, id in files_ids: @@ -31,7 +32,7 @@ class UnZip(AbtractExtractor): return result - def extract(self, progress, password=None): + def extract(self, progress, password=""): try: z = zipfile.ZipFile(self.file) self.files = z.namelist() -- cgit v1.2.3 From 6d9c5ff2e06e06e33fa6674f013dd0e290092bbd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 20 Dec 2014 14:25:57 +0100 Subject: [SimpleHoster] Improve checkFile --- module/plugins/internal/SimpleHoster.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 949a6c66a..784ef8c85 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -155,7 +155,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.76" + __version__ = "0.77" __pattern__ = r'^unmatchable$' @@ -407,11 +407,16 @@ class SimpleHoster(Hoster): def downloadLink(self, link): if link and isinstance(link, basestring): + self.correctCaptcha() self.download(link, disposition=True) def checkFile(self): - if not self.lastDownload or not exists(fs_encode(self.lastDownload)): + if self.cTask and not self.lastDownload: + self.invalidCaptcha() + self.retry(10, reason=_("Wrong captcha")) + + elif not self.lastDownload or not exists(fs_encode(self.lastDownload)): self.fail(_("No file downloaded")) else: -- cgit v1.2.3 From ad8d5997b60012e7f1c895956f5ea713870177bb Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 23 Dec 2014 00:58:03 +0100 Subject: [XFSHoster] Improve error handling --- module/plugins/internal/XFSHoster.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 622f34ea9..7038743c1 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.29" + __version__ = "0.30" __pattern__ = r'^unmatchable$' @@ -238,7 +238,7 @@ class XFSHoster(SimpleHoster): elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: self.tempOffline() - elif 'download files up to' in self.errmsg: + elif 'up to' in self.errmsg: self.fail(_("File too large for free download")) else: -- cgit v1.2.3 From 57a319563651a07bf8265ab52da0d7375191319c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 23 Dec 2014 13:13:42 +0100 Subject: Rename MultiHoster plugin to MultiHook --- module/plugins/internal/MultiHook.py | 217 +++++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 module/plugins/internal/MultiHook.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py new file mode 100644 index 000000000..dcf1c3383 --- /dev/null +++ b/module/plugins/internal/MultiHook.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- + +import re + +from module.plugins.Hook import Hook +from module.utils import remove_chars + + +class MultiHook(Hook): + __name__ = "MultiHook" + __type__ = "hook" + __version__ = "0.22" + + __description__ = """Hook plugin for MultiHoster""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org")] + + + interval = 12 * 60 * 60 #: reload hosters every 12h + + HOSTER_REPLACEMENTS = [("1fichier.com" , "onefichier.com"), + ("2shared.com" , "twoshared.com" ), + ("4shared.com" , "fourshared.com"), + ("cloudnator.com" , "shragle.com" ), + ("easy-share.com" , "crocko.com" ), + ("fileparadox.com", "fileparadox.in"), + ("freakshare.net" , "freakshare.com"), + ("hellshare.com" , "hellshare.cz" ), + ("ifile.it" , "filecloud.io" ), + ("nowdownload.ch" , "nowdownload.sx"), + ("nowvideo.co" , "nowvideo.sx" ), + ("putlocker.com" , "firedrive.com" ), + ("share-rapid.cz" , "multishare.cz" ), + ("sharerapid.cz" , "multishare.cz" ), + ("ul.to" , "uploaded.to" ), + ("uploaded.net" , "uploaded.to" )] + HOSTER_EXCLUDED = [] + + + def setup(self): + self.hosters = [] + self.supported = [] + self.new_supported = [] + + + def getConfig(self, option, default=''): + """getConfig with default value - sublass may not implements all config options""" + try: + return self.getConf(option) + except KeyError: + return default + + + def getHosterCached(self): + if not self.hosters: + try: + hosterSet = self.toHosterSet(self.getHoster()) - set(self.HOSTER_EXCLUDED) + except Exception, e: + self.logError(e) + return [] + + try: + configMode = self.getConfig('hosterListMode', 'all') + if configMode in ("listed", "unlisted"): + configSet = self.toHosterSet(self.getConfig('hosterList', '').replace('|', ',').replace(';', ',').split(',')) + + if configMode == "listed": + hosterSet &= configSet + else: + hosterSet -= configSet + + except Exception, e: + self.logError(e) + + self.hosters = list(hosterSet) + + return self.hosters + + + def toHosterSet(self, hosters): + hosters = set((str(x).strip().lower() for x in hosters)) + + for rep in self.HOSTER_REPLACEMENTS: + if rep[0] in hosters: + hosters.remove(rep[0]) + hosters.add(rep[1]) + + hosters.discard('') + return hosters + + + def getHoster(self): + """Load list of supported hoster + + :return: List of domain names + """ + raise NotImplementedError + + + def coreReady(self): + if self.cb: + self.core.scheduler.removeJob(self.cb) + + self.setConfig("activated", True) #: config not in sync after plugin reload + + cfg_interval = self.getConfig("interval", None) #: reload interval in hours + if cfg_interval is not None: + self.interval = cfg_interval * 60 * 60 + + if self.interval: + self._periodical() + else: + self.periodical() + + + def initPeriodical(self): + pass + + + def periodical(self): + """reload hoster list periodically""" + self.logInfo(_("Reloading supported hoster list")) + + old_supported = self.supported + self.supported = [] + self.new_supported = [] + self.hosters = [] + + self.overridePlugins() + + old_supported = [hoster for hoster in old_supported if hoster not in self.supported] + if old_supported: + self.logDebug("UNLOAD", ", ".join(old_supported)) + for hoster in old_supported: + self.unloadHoster(hoster) + + + def overridePlugins(self): + pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys()) + accountList = [name.lower() for name, data in self.core.accountManager.accounts.iteritems() if data] + excludedList = [] + + for hoster in self.getHosterCached(): + name = remove_chars(hoster, "-.") + + if name in accountList: + excludedList.append(hoster) + else: + if name in pluginMap: + self.supported.append(pluginMap[name]) + else: + self.new_supported.append(hoster) + + if not self.supported and not self.new_supported: + self.logError(_("No Hoster loaded")) + return + + module = self.core.pluginManager.getPlugin(self.__name__) + klass = getattr(module, self.__name__) + + # inject plugin plugin + self.logDebug("Overwritten Hosters", ", ".join(sorted(self.supported))) + for hoster in self.supported: + hdict = self.core.pluginManager.hosterPlugins[hoster] + hdict['new_module'] = module + hdict['new_name'] = self.__name__ + + if excludedList: + self.logInfo(_("The following hosters were not overwritten - account exists"), ", ".join(sorted(excludedList))) + + if self.new_supported: + hosters = sorted(self.new_supported) + + self.logDebug("New Hosters", ", ".join(hosters)) + + # create new regexp + regexp = r'.*(%s).*' % "|".join([x.replace(".", "\.") for x in hosters]) + if hasattr(klass, "__pattern__") and isinstance(klass.__pattern__, basestring) and '://' in klass.__pattern__: + regexp = r'%s|%s' % (klass.__pattern__, regexp) + + self.logDebug("Regexp", regexp) + + hdict = self.core.pluginManager.hosterPlugins[self.__name__] + hdict['pattern'] = regexp + hdict['re'] = re.compile(regexp) + + + def unloadHoster(self, hoster): + hdict = self.core.pluginManager.hosterPlugins[hoster] + if "module" in hdict: + del hdict['module'] + + if "new_module" in hdict: + del hdict['new_module'] + del hdict['new_name'] + + + def unload(self): + """Remove override for all hosters. Scheduler job is removed by hookmanager""" + for hoster in self.supported: + self.unloadHoster(hoster) + + # reset pattern + klass = getattr(self.core.pluginManager.getPlugin(self.__name__), self.__name__) + hdict = self.core.pluginManager.hosterPlugins[self.__name__] + hdict['pattern'] = getattr(klass, "__pattern__", r'^unmatchable$') + hdict['re'] = re.compile(hdict['pattern']) + + + def downloadFailed(self, pyfile): + """remove plugin override if download fails but not if file is offline/temp.offline""" + if pyfile.hasStatus("failed") and self.getConfig("unloadFailing", True): + hdict = self.core.pluginManager.hosterPlugins[pyfile.pluginname] + if "new_name" in hdict and hdict['new_name'] == self.__name__: + self.logDebug("Unload MultiHook", pyfile.pluginname, hdict) + self.unloadHoster(pyfile.pluginname) + pyfile.setStatus("queued") -- cgit v1.2.3 From 87203e996fb42c172b15e29f0e394d5b328d9ac2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 23 Dec 2014 13:20:53 +0100 Subject: New plugin: MultiHoster --- module/plugins/internal/MultiHoster.py | 226 ++++++--------------------------- 1 file changed, 36 insertions(+), 190 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 68550a4dd..4425cfce0 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -2,216 +2,62 @@ import re -from module.plugins.Hook import Hook -from module.utils import remove_chars +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies -class MultiHoster(Hook): +class MultiHoster(SimpleHoster): __name__ = "MultiHoster" - __type__ = "hook" - __version__ = "0.22" + __type__ = "hoster" + __version__ = "0.23" - __description__ = """Generic MultiHoster plugin""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] - - - interval = 12 * 60 * 60 #: reload hosters every 12h - - HOSTER_REPLACEMENTS = [("1fichier.com" , "onefichier.com"), - ("2shared.com" , "twoshared.com" ), - ("4shared.com" , "fourshared.com"), - ("cloudnator.com" , "shragle.com" ), - ("easy-share.com" , "crocko.com" ), - ("fileparadox.com", "fileparadox.in"), - ("freakshare.net" , "freakshare.com"), - ("hellshare.com" , "hellshare.cz" ), - ("ifile.it" , "filecloud.io" ), - ("nowdownload.ch" , "nowdownload.sx"), - ("nowvideo.co" , "nowvideo.sx" ), - ("putlocker.com" , "firedrive.com" ), - ("share-rapid.cz" , "multishare.cz" ), - ("sharerapid.cz" , "multishare.cz" ), - ("ul.to" , "uploaded.to" ), - ("uploaded.net" , "uploaded.to" )] - HOSTER_EXCLUDED = [] - - - def setup(self): - self.hosters = [] - self.supported = [] - self.new_supported = [] - - - def getConfig(self, option, default=''): - """getConfig with default value - sublass may not implements all config options""" - try: - return self.getConf(option) - except KeyError: - return default - - - def getHosterCached(self): - if not self.hosters: - try: - hosterSet = self.toHosterSet(self.getHoster()) - set(self.HOSTER_EXCLUDED) - except Exception, e: - self.logError(e) - return [] - - try: - configMode = self.getConfig('hosterListMode', 'all') - if configMode in ("listed", "unlisted"): - configSet = self.toHosterSet(self.getConfig('hosterList', '').replace('|', ',').replace(';', ',').split(',')) - - if configMode == "listed": - hosterSet &= configSet - else: - hosterSet -= configSet + __pattern__ = r'^unmatchable$' - except Exception, e: - self.logError(e) - - self.hosters = list(hosterSet) - - return self.hosters - - - def toHosterSet(self, hosters): - hosters = set((str(x).strip().lower() for x in hosters)) - - for rep in self.HOSTER_REPLACEMENTS: - if rep[0] in hosters: - hosters.remove(rep[0]) - hosters.add(rep[1]) + __description__ = """Multi hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - hosters.discard('') - return hosters + DIRECT_LINK = True + MULTI_HOSTER = True + LOGIN_ACCOUNT = True + LOGIN_PREMIUM = False - def getHoster(self): - """Load list of supported hoster - :return: List of domain names - """ - raise NotImplementedError + def setup(self): + self.chunkLimit = 1 + self.multiDL = self.premium - def coreReady(self): - if self.cb: - self.core.scheduler.removeJob(self.cb) + def prepare(self): + self.info = {} + self.link = "" + self.multihost = False - self.setConfig("activated", True) #: config not in sync after plugin reload + self.req.setOption("timeout", 120) - cfg_interval = self.getConfig("interval", None) #: reload interval in hours - if cfg_interval is not None: - self.interval = cfg_interval * 60 * 60 + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) - if self.interval: - self._periodical() + if self.DIRECT_LINK is None: + self.directDL = bool(self.account) else: - self.periodical() - - - def initPeriodical(self): - pass + self.directDL = self.DIRECT_LINK + if (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] + or re.match(self.__pattern__, self.pyfile.url) is None): - def periodical(self): - """reload hoster list periodically""" - self.logInfo(_("Reloading supported hoster list")) + if self.LOGIN_ACCOUNT and not self.account: + self.logError(_("Required account not found")) - old_supported = self.supported - self.supported = [] - self.new_supported = [] - self.hosters = [] + elif self.LOGIN_PREMIUM and not self.premium: + self.logError(_("Required premium account not found")) - self.overridePlugins() - - old_supported = [hoster for hoster in old_supported if hoster not in self.supported] - if old_supported: - self.logDebug("UNLOAD", ", ".join(old_supported)) - for hoster in old_supported: - self.unloadHoster(hoster) - - - def overridePlugins(self): - pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys()) - accountList = [name.lower() for name, data in self.core.accountManager.accounts.iteritems() if data] - excludedList = [] - - for hoster in self.getHosterCached(): - name = remove_chars(hoster, "-.") - - if name in accountList: - excludedList.append(hoster) else: - if name in pluginMap: - self.supported.append(pluginMap[name]) - else: - self.new_supported.append(hoster) - - if not self.supported and not self.new_supported: - self.logError(_("No Hoster loaded")) - return - - module = self.core.pluginManager.getPlugin(self.__name__) - klass = getattr(module, self.__name__) - - # inject plugin plugin - self.logDebug("Overwritten Hosters", ", ".join(sorted(self.supported))) - for hoster in self.supported: - hdict = self.core.pluginManager.hosterPlugins[hoster] - hdict['new_module'] = module - hdict['new_name'] = self.__name__ - - if excludedList: - self.logInfo(_("The following hosters were not overwritten - account exists"), ", ".join(sorted(excludedList))) - - if self.new_supported: - hosters = sorted(self.new_supported) + self.multihost = True - self.logDebug("New Hosters", ", ".join(hosters)) + self.pyfile.url = replace_patterns(self.pyfile.url, + self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 - # create new regexp - regexp = r'.*(%s).*' % "|".join([x.replace(".", "\.") for x in hosters]) - if hasattr(klass, "__pattern__") and isinstance(klass.__pattern__, basestring) and '://' in klass.__pattern__: - regexp = r'%s|%s' % (klass.__pattern__, regexp) - self.logDebug("Regexp", regexp) - - hdict = self.core.pluginManager.hosterPlugins[self.__name__] - hdict['pattern'] = regexp - hdict['re'] = re.compile(regexp) - - - def unloadHoster(self, hoster): - hdict = self.core.pluginManager.hosterPlugins[hoster] - if "module" in hdict: - del hdict['module'] - - if "new_module" in hdict: - del hdict['new_module'] - del hdict['new_name'] - - - def unload(self): - """Remove override for all hosters. Scheduler job is removed by hookmanager""" - for hoster in self.supported: - self.unloadHoster(hoster) - - # reset pattern - klass = getattr(self.core.pluginManager.getPlugin(self.__name__), self.__name__) - hdict = self.core.pluginManager.hosterPlugins[self.__name__] - hdict['pattern'] = getattr(klass, "__pattern__", r'^unmatchable$') - hdict['re'] = re.compile(hdict['pattern']) - - - def downloadFailed(self, pyfile): - """remove plugin override if download fails but not if file is offline/temp.offline""" - if pyfile.hasStatus("failed") and self.getConfig("unloadFailing", True): - hdict = self.core.pluginManager.hosterPlugins[pyfile.pluginname] - if "new_name" in hdict and hdict['new_name'] == self.__name__: - self.logDebug("Unload MultiHoster", pyfile.pluginname, hdict) - self.unloadHoster(pyfile.pluginname) - pyfile.setStatus("queued") + def handleMulti(self): + raise NotImplementedError -- cgit v1.2.3 From e1d4186c62512d8bb76d35b6f8d1828d8d0aa94e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 23 Dec 2014 13:26:32 +0100 Subject: [SimpleHoster] Improve multi-hoster feature --- module/plugins/internal/SimpleHoster.py | 38 ++++++++++++++++----------------- module/plugins/internal/XFSHoster.py | 5 ++++- 2 files changed, 23 insertions(+), 20 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 784ef8c85..ab114c9ec 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -155,7 +155,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.77" + __version__ = "0.78" __pattern__ = r'^unmatchable$' @@ -214,9 +214,9 @@ class SimpleHoster(Hoster): TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct COOKIES = True #: or False or list of tuples [(domain, name, value)] - FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - CHECK_DIRECT_LINK = None #: Set to True to check for direct link, set to None to do it only if self.account is True - MULTI_HOSTER = False #: Set to True to leech other hoster link (according its multihoster hook if available) + CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + DIRECT_LINK = None #: Set to True to looking for direct link (as defined in handleDirect method), set to None to do it if self.account is True else False + MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) @classmethod @@ -334,17 +334,13 @@ class SimpleHoster(Hoster): if (self.MULTI_HOSTER and (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] or re.match(self.__pattern__, self.pyfile.url) is None)): + self.multihost = True + return - self.logInfo("Multi hoster detected") - - if self.account: - self.multihost = True - return - else: - self.fail(_("Only registered or premium users can use url leech feature")) - - if self.CHECK_DIRECT_LINK is None: + if self.DIRECT_LINK is None: self.directDL = bool(self.account) + else: + self.directDL = self.DIRECT_LINK self.pyfile.url = replace_patterns(self.pyfile.url, self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 @@ -360,16 +356,20 @@ class SimpleHoster(Hoster): def process(self, pyfile): self.prepare() - if self.multihost: + if self.directDL: + self.logDebug("Looking for direct download link...") + self.handleDirect() + + if self.multihost and not self.link and not self.lastDownload: self.logDebug("Looking for leeched download link...") self.logDebug("File url: %s" % pyfile.url) self.handleMulti() - elif self.directDL: - self.logDebug("Looking for direct download link...") - self.handleDirect() + if not self.link and not self.lastDownload: + self.MULTI_HOSTER = False + self.retry(1, reason="Multi hoster fails") - if not self.link: + if not self.link and not self.lastDownload: self.preload() if self.html is None: @@ -390,7 +390,7 @@ class SimpleHoster(Hoster): self.checkErrors() - if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): self.logDebug("Handled as premium download") self.handlePremium() diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 7038743c1..dd2dd2527 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.30" + __version__ = "0.31" __pattern__ = r'^unmatchable$' @@ -142,6 +142,9 @@ class XFSHoster(SimpleHoster): def handleMulti(self): + if not self.account: + self.fail(_("Only registered or premium users can use url leech feature")) + #only tested with easybytez.com self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) -- cgit v1.2.3 From badfd177fb3639869441f4738461c7f576e1d078 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 23 Dec 2014 17:22:47 +0100 Subject: [SimpleHoster] Update --- module/plugins/internal/SimpleCrypter.py | 12 +---- module/plugins/internal/SimpleHoster.py | 81 +++++++++++++++----------------- 2 files changed, 38 insertions(+), 55 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 53ffaf4a6..e0dd10585 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -12,7 +12,7 @@ from module.utils import fixup class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.32" + __version__ = "0.33" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -120,16 +120,6 @@ class SimpleCrypter(Crypter, SimpleHoster): self.packages = [(self.info['name'], self.links, self.info['folder'])] - def checkStatus(self): - status = self.info['status'] - - if status is 1: - self.offline() - - elif status is 6: - self.tempOffline() - - def checkNameSize(self): name = self.info['name'] url = self.info['url'] diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index ab114c9ec..95833d0cf 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -125,29 +125,34 @@ def timestamp(): #@TODO: Move to hoster class in 0.4.10 -def _isDirectLink(self, url, resumable=True): - header = self.load(url, ref=True, just_header=True, decode=True) +def _isDirectLink(self, url, resumable=False): + link = "" - if not 'location' in header or not header['location']: - return "" + for i in xrange(5 if resumable else 1): + header = self.load(url, ref=True, cookies=True, just_header=True, decode=True) - location = header['location'] + if 'content-disposition' in header: + link = url - resumable = False #@NOTE: Testing... + elif 'location' in header and header['location']: + location = header['location'] - if resumable: #: sometimes http code may be wrong... - if 'location' in self.load(location, ref=True, cookies=True, just_header=True, decode=True): - return "" - else: - if not 'code' in header or header['code'] != 302: - return "" + if not urlparse(location).scheme: + p = urlparse(url) + base = "%s://%s" % (p.scheme, p.netloc) + location = urljoin(base, location) + + if 'code' in header and header['code'] == 302: + link = location + + elif resumable: + url = location + self.logDebug("Redirect #%d to: %s" % (++i, location)) + continue - if urlparse(location).scheme: - link = location + break else: - p = urlparse(url) - base = "%s://%s" % (p.scheme, p.netloc) - link = urljoin(base, location) + self.logError(_("Too many redirects")) return link @@ -155,7 +160,7 @@ def _isDirectLink(self, url, resumable=True): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.78" + __version__ = "0.79" __pattern__ = r'^unmatchable$' @@ -355,6 +360,7 @@ class SimpleHoster(Hoster): def process(self, pyfile): self.prepare() + self.checkInfo() if self.directDL: self.logDebug("Looking for direct download link...") @@ -371,32 +377,15 @@ class SimpleHoster(Hoster): if not self.link and not self.lastDownload: self.preload() + self.checkInfo() if self.html is None: self.fail(_("No html retrieved")) - self.updateInfo(self.getInfo(pyfile.url, self.html)) - - self.checkNameSize() - - if hasattr(self, 'PREMIUM_ONLY_PATTERN'): - premium_only = re.search(self.PREMIUM_ONLY_PATTERN, self.html) - else: - premium_only = False - - #: Usually premium only pages doesn't show any file information - if not premium_only: - self.checkStatus() - - self.checkErrors() - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): self.logDebug("Handled as premium download") self.handlePremium() - elif premium_only: - self.fail(_("Link require a premium account to be handled")) - else: self.logDebug("Handled as free download") self.handleFree() @@ -432,6 +421,9 @@ class SimpleHoster(Hoster): def checkErrors(self): + if hasattr(self, 'PREMIUM_ONLY_PATTERN') and self.premium and re.search(self.PREMIUM_ONLY_PATTERN, self.html): + self.fail(_("Link require a premium account to be handled")) + if hasattr(self, 'ERROR_PATTERN'): m = re.search(self.ERROR_PATTERN, self.html) if m: @@ -459,9 +451,8 @@ class SimpleHoster(Hoster): self.tempOffline() elif status is not 2: - self.logInfo(_("File status: %s") % statusMap[status], - _("File info: %s") % self.info) - self.error(_("No file info retrieved")) + self.logDebug(_("File status: %s") % statusMap[status], + _("File info: %s") % self.info) def checkNameSize(self): @@ -484,9 +475,14 @@ class SimpleHoster(Hoster): def checkInfo(self): - self.checkErrors() + self.updateInfo(self.getInfo(self.pyfile.url, self.html)) - self.updateInfo(self.getInfo(self.pyfile.url, self.html or "")) + self.checkNameSize() + + if self.html: + self.checkErrors() + + self.updateInfo(self.getInfo(self.pyfile.url, self.html)) self.checkNameSize() self.checkStatus() @@ -512,9 +508,6 @@ class SimpleHoster(Hoster): self.logInfo(_("Direct download link detected")) self.link = link - - self.updateInfo(self.getInfo(self.pyfile.url)) - self.checkNameSize() else: self.logDebug(_("Direct download link not found")) -- cgit v1.2.3 From cc833c564e01971b0a915c2f5343ad683c65b825 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 23 Dec 2014 18:49:39 +0100 Subject: [LinestorageCom] Fix account plugin --- module/plugins/internal/XFSAccount.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSAccount.py b/module/plugins/internal/XFSAccount.py index 2094b1480..2784ecd0b 100644 --- a/module/plugins/internal/XFSAccount.py +++ b/module/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from module.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.32" + __version__ = "0.33" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -44,11 +44,11 @@ class XFSAccount(Account): def init(self): - # if not self.HOSTER_DOMAIN: - # self.fail(_("Missing HOSTER_DOMAIN")) + if not self.HOSTER_DOMAIN: + self.logError(_("Missing HOSTER_DOMAIN")) if not self.HOSTER_URL: - self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN + self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN or "" def loadAccountInfo(self, user, req): -- cgit v1.2.3 From 9c9843e6c7681ef52a882033eaabb44afacc5b8e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 23 Dec 2014 18:51:01 +0100 Subject: [SimpleHoster] Code cosmetics --- module/plugins/internal/SimpleHoster.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 95833d0cf..6726726e1 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -368,7 +368,6 @@ class SimpleHoster(Hoster): if self.multihost and not self.link and not self.lastDownload: self.logDebug("Looking for leeched download link...") - self.logDebug("File url: %s" % pyfile.url) self.handleMulti() if not self.link and not self.lastDownload: @@ -451,8 +450,8 @@ class SimpleHoster(Hoster): self.tempOffline() elif status is not 2: - self.logDebug(_("File status: %s") % statusMap[status], - _("File info: %s") % self.info) + self.logDebug("File status: %s" % statusMap[status], + "File info: %s" % self.info) def checkNameSize(self): @@ -496,9 +495,9 @@ class SimpleHoster(Hoster): def updateInfo(self, info): - self.logDebug(_("File info (before update): %s") % self.info) + self.logDebug(_("File info (BEFORE): %s") % self.info) self.info.update(info) - self.logDebug(_("File info (after update): %s") % self.info) + self.logDebug(_("File info (AFTER): %s") % self.info) def handleDirect(self): -- cgit v1.2.3 From a4786e340993bbfc5d2bf971c9bec18863d3dd80 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 23 Dec 2014 19:29:15 +0100 Subject: [MultiHoster] Update --- module/plugins/internal/MultiHoster.py | 53 +++++++++++++++------------------- 1 file changed, 24 insertions(+), 29 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 4425cfce0..e5c28d034 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -2,13 +2,13 @@ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.23" + __version__ = "0.24" __pattern__ = r'^unmatchable$' @@ -18,9 +18,6 @@ class MultiHoster(SimpleHoster): DIRECT_LINK = True - MULTI_HOSTER = True - LOGIN_ACCOUNT = True - LOGIN_PREMIUM = False def setup(self): @@ -28,36 +25,34 @@ class MultiHoster(SimpleHoster): self.multiDL = self.premium - def prepare(self): - self.info = {} - self.link = "" - self.multihost = False + def process(self, pyfile): + self.prepare() - self.req.setOption("timeout", 120) + if self.directDL: + self.logDebug("Looking for direct download link...") + self.handleDirect() - if isinstance(self.COOKIES, list): - set_cookies(self.req.cj, self.COOKIES) + if not self.link and not self.lastDownload: + self.preload() - if self.DIRECT_LINK is None: - self.directDL = bool(self.account) - else: - self.directDL = self.DIRECT_LINK + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as premium download") + self.handlePremium() - if (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] - or re.match(self.__pattern__, self.pyfile.url) is None): + else: + self.logDebug("Handled as free download") + self.handleFree() - if self.LOGIN_ACCOUNT and not self.account: - self.logError(_("Required account not found")) + self.downloadLink(self.link) + self.checkFile() - elif self.LOGIN_PREMIUM and not self.premium: - self.logError(_("Required premium account not found")) - else: - self.multihost = True + def handlePremium(self): + return self.handleFree() - self.pyfile.url = replace_patterns(self.pyfile.url, - self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 - - def handleMulti(self): - raise NotImplementedError + def handleFree(self): + if self.premium: + raise NotImplementedError + else: + self.logError(_("Required account not found")) -- cgit v1.2.3 From 29df1397bbbe80eced4674b6fa39e16540c80901 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 24 Dec 2014 01:11:44 +0100 Subject: Extractor rewritten --- module/plugins/internal/UnRar.py | 179 +++++++++++++++++++++++---------------- module/plugins/internal/UnZip.py | 64 ++++++++++---- 2 files changed, 157 insertions(+), 86 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 4bbd2042c..5633b31f7 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -4,11 +4,11 @@ import os import re from glob import glob -from os.path import basename, join +from os.path import basename, dirname, join from string import digits from subprocess import Popen, PIPE -from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError +from module.plugins.internal.AbstractExtractor import AbtractExtractor, PasswordError, ArchiveError, CRCError from module.utils import save_join, decode @@ -22,21 +22,25 @@ def renice(pid, value): class UnRar(AbtractExtractor): __name__ = "UnRar" - __version__ = "0.21" + __version__ = "1.00" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] CMD = "unrar" + EXTENSIONS = ["rar", "zip", "cab", "arj", "lzh", "tar", "gz", "bz2", "ace", "uue", "jar", "iso", "7z", "xz", "z"] + + #@NOTE: there are some more uncovered rar formats - re_splitfile = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) - re_partfiles = re.compile(r'.*\.(rar|r\d+)', re.I) + re_rarpart = re.compile(r'(.*)\.part(\d+)\.rar$', re.I) + re_rarfile = re.compile(r'.*\.(rar|r\d+)$', re.I) + re_filelist = re.compile(r'(.+)\s+(\d+)\s+(\d+)\s+|(.+)\s+(\d+)\s+\d\d-\d\d-\d\d\s+\d\d:\d\d\s+(.+)') - re_wrongpwd = re.compile(r'(Corrupt file or wrong password|password incorrect)', re.I) + re_wrongpwd = re.compile(r'password', re.I) + re_wrongcrc = re.compile(r'encrypted|damaged|CRC failed|checksum error', re.I) @classmethod @@ -59,70 +63,100 @@ class UnRar(AbtractExtractor): return True + @classmethod + def isArchive(cls, file): + f = basename(file).lower() + return any(f.endswith('.%s' % ext) for ext in cls.EXTENSIONS) + + @classmethod def getTargets(cls, files_ids): - result = [] + targets = [] for file, id in files_ids: - if not file.endswith(".rar"): + if not cls.isArchive(file): continue - match = cls.re_splitfile.findall(file) - if match: + m = cls.re_rarpart.findall(file) + if m: # only add first parts - if int(match[0][1]) == 1: - result.append((file, id)) + if int(m[0][1]) == 1: + targets.append((file, id)) else: - result.append((file, id)) + targets.append((file, id)) - return result + return targets - def init(self): - self.passwordProtected = False - self.headerProtected = False #: list files will not work without password - self.password = "" #: save the correct password + def check(self, out="", err=""): + if not out or not err: + return + if err.strip(): + if self.re_wrongpwd.search(err): + raise PasswordError - def checkArchive(self): - p = self.call_unrar("l", "-v", self.file) - out, err = p.communicate() - if self.re_wrongpwd.search(err): - self.passwordProtected = True - self.headerProtected = True - return True + elif self.re_wrongcrc.search(err): + raise CRCError + + else: #: raise error if anything is on stderr + raise ArchiveError(err.strip()) # output only used to check if passworded files are present for attr in self.re_filelist.findall(out): if attr[0].startswith("*"): - self.passwordProtected = True - return True + raise PasswordError + - self.listContent() - if not self.files: - raise ArchiveError("Empty Archive") + def verify(self): + p = self.call_cmd("l", "-v", self.file, password=self.password) + + self.check(*p.communicate()) + + if p and p.returncode: + raise ArchiveError("Process terminated") + + if not self.list(): + raise ArchiveError("Empty archive") + + + def isPassword(self, password): + if isinstance(password, basestring): + p = self.call_cmd("l", "-v", self.file, password=password) + out, err = p.communicate() + + if not self.re_wrongpwd.search(err): + return True return False - def checkPassword(self, password): - # at this point we can only verify header protected files - if self.headerProtected: - p = self.call_unrar("l", "-v", self.file, password=password) + def repair(self): + p = self.call_cmd("rc", self.file) + out, err = p.communicate() + + if p.returncode or err.strip(): + p = self.call_cmd("r", self.file) out, err = p.communicate() - if self.re_wrongpwd.search(err): + + if p.returncode or err.strip(): return False + else: + self.file = join(dirname(self.file), re.search(r'(fixed|rebuild)\.%s' % basename(self.file), out).group(0)) return True - def extract(self, progress, password=""): + def extract(self, progress=lambda x: None): + self.verify() + + progress(0) + command = "x" if self.fullpath else "e" - p = self.call_unrar(command, self.file, self.out, password=password) - renice(p.pid, self.renice) + p = self.call_cmd(command, self.file, self.out, password=self.password) - progress(0) + renice(p.pid, self.renice) progressstring = "" while True: @@ -131,7 +165,7 @@ class UnRar(AbtractExtractor): if not c: break # reading a percentage sign -> set progress and restart - if c == '%': + if c is '%': progress(int(progressstring)) progressstring = "" # not reading a digit -> therefore restart @@ -139,46 +173,43 @@ class UnRar(AbtractExtractor): progressstring = "" # add digit to progressstring else: - progressstring = progressstring + c + progressstring += c progress(100) - # retrieve stderr - err = p.stderr.read() + self.files = self.list() - if "CRC failed" in err and not password and not self.passwordProtected: - raise CRCError - elif "CRC failed" in err: - raise WrongPassword + # retrieve stderr + self.check(err=p.stderr.read()) - if err.strip(): #: raise error if anything is on stderr - raise ArchiveError(err.strip()) if p.returncode: raise ArchiveError("Process terminated") - if not self.files: - self.password = password - self.listContent() - def getDeleteFiles(self): if ".part" in basename(self.file): return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.I)) + # get files which matches .r* and filter unsuited files out parts = glob(re.sub(r"(?<=\.r)ar$", "*", self.file, re.I)) - return filter(lambda x: self.re_partfiles.match(x), parts) + return filter(lambda x: self.re_rarfile.match(x), parts) - def listContent(self): + + def list(self): command = "vb" if self.fullpath else "lb" - p = self.call_unrar(command, "-v", self.file, password=self.password) + + p = self.call_cmd(command, "-v", self.file, password=self.password) out, err = p.communicate() - if "Cannot open" in err: - raise ArchiveError("Cannot open file") + if err.strip(): + self.m.logError(err) + if "Cannot open" in err: + return list() - if err.strip(): #: only log error at this point - self.m.logError(err.strip()) + if p.returncode: + self.m.logError("Process terminated") + return list() result = set() @@ -186,17 +217,22 @@ class UnRar(AbtractExtractor): f = f.strip() result.add(save_join(self.out, f)) - self.files = result + return list(result) - def call_unrar(self, command, *xargs, **kwargs): + def call_cmd(self, command, *xargs, **kwargs): args = [] + # overwrite flag - args.append("-o+") if self.overwrite else args.append("-o-") + if self.overwrite: + args.append("-o+") + else: + args.append("-o-") + if self.delete: + args.append("-or") - if self.excludefiles: - for word in self.excludefiles.split(';'): - args.append("-x%s" % word) + for word in self.excludefiles: + args.append("-x%s" % word.strip()) # assume yes on all queries args.append("-y") @@ -207,10 +243,11 @@ class UnRar(AbtractExtractor): else: args.append("-p-") + if self.keepbroken: + args.append("-kb") + # NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) self.m.logDebug(" ".join(call)) - p = Popen(call, stdout=PIPE, stderr=PIPE) - - return p + return Popen(call, stdout=PIPE, stderr=PIPE) diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 81c298784..b3d54cba0 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -1,19 +1,23 @@ # -*- coding: utf-8 -*- +from __future__ import with_statement + import sys import zipfile -from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError +from module.plugins.internal.AbstractExtractor import AbtractExtractor, PasswordError, ArchiveError, CRCError class UnZip(AbtractExtractor): __name__ = "UnZip" - __version__ = "0.12" + __version__ = "1.00" __description__ = """Zip extractor plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + EXTENSIONS = ["zip", "zip64"] @classmethod @@ -22,31 +26,61 @@ class UnZip(AbtractExtractor): @classmethod - def getTargets(cls, files_ids): - result = [] + def isArchive(cls, file): + return zipfile.is_zipfile(file) - for file, id in files_ids: - if file.endswith(".zip"): - result.append((file, id)) - return result + def verify(self): + try: + with zipfile.ZipFile(self.file, 'r', allowZip64=True) as z: + z.setpassword(self.password) + badcrc = z.testzip() + except (BadZipfile, LargeZipFile), e: + raise ArchiveError(e) - def extract(self, progress, password=""): + except RuntimeError, e: + if 'encrypted' in e: + raise PasswordError + else: + raise ArchiveError(e) + + else: + if badcrc: + raise CRCError + + if not self.list(): + raise ArchiveError("Empty archive") + + + def list(self): try: - z = zipfile.ZipFile(self.file) - self.files = z.namelist() - z.extractall(self.out, pwd=password) + with zipfile.ZipFile(self.file, 'r', allowZip64=True) as z: + z.setpassword(self.password) + return z.namelist() + except Exception: + return list() + + + def extract(self, progress=lambda x: None): + try: + with zipfile.ZipFile(self.file, 'r', allowZip64=True) as z: + progress(0) + z.extractall(self.out, pwd=self.password) + progress(100) except (BadZipfile, LargeZipFile), e: raise ArchiveError(e) except RuntimeError, e: if e is "Bad password for file": - raise WrongPassword + raise PasswordError else: raise ArchiveError(e) + finally: + self.files = self.list() + def getDeleteFiles(self): return [self.file] -- cgit v1.2.3 From deea9c544157037ad8c4c55d61ea92b3696b117b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 24 Dec 2014 01:12:17 +0100 Subject: Extractor rewritten (2) --- module/plugins/internal/AbstractExtractor.py | 55 ++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 12 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py index c8a73c861..310897d4e 100644 --- a/module/plugins/internal/AbstractExtractor.py +++ b/module/plugins/internal/AbstractExtractor.py @@ -8,13 +8,13 @@ class CRCError(Exception): pass -class WrongPassword(Exception): +class PasswordError(Exception): pass class AbtractExtractor: __name__ = "AbtractExtractor" - __version__ = "0.11" + __version__ = "0.12" __description__ = """Abtract extractor plugin""" __license__ = "GPLv3" @@ -22,6 +22,9 @@ class AbtractExtractor: ("Walter Purcaro", "vuolter@gmail.com")] + EXTENSIONS = [] + + @classmethod def checkDeps(cls): """ Check if system statisfy dependencies @@ -30,16 +33,27 @@ class AbtractExtractor: return True + @classmethod + def isArchive(cls, file): + raise NotImplementedError + + @classmethod def getTargets(cls, files_ids): """ Filter suited targets from list of filename id tuple list :param files_ids: List of filepathes :return: List of targets, id tuple list """ - raise NotImplementedError + targets = [] + + for file, id in files_ids: + if cls.isArchive(file): + targets.append((file, id)) + + return targets - def __init__(self, m, file, out, fullpath, overwrite, excludefiles, renice): + def __init__(self, m, file, out, password, fullpath, overwrite, excludefiles, renice, delete, keepbroken): """Initialize extractor for specific file :param m: ExtractArchive Hook plugin @@ -52,10 +66,13 @@ class AbtractExtractor: self.m = m self.file = file self.out = out + self.password = password self.fullpath = fullpath self.overwrite = overwrite self.excludefiles = excludefiles self.renice = renice + self.delete = delete + self.keepbroken = keepbroken self.files = [] #: Store extracted files here @@ -64,36 +81,50 @@ class AbtractExtractor: pass - def checkArchive(self): + def verify(self): """Check if password if needed. Raise ArchiveError if integrity is questionable. - :return: boolean :raises ArchiveError """ - return False + pass - def checkPassword(self, password): + def isPassword(self, password): """ Check if the given password is/might be correct. If it can not be decided at this point return true. :param password: :return: boolean """ - return True + if isinstance(password, basestring): + return True + else: + return False + + + def setPassword(self, password): + if self.isPassword(password): + self.password = password + return True + else: + return False + + + def repair(self): + return False - def extract(self, progress, password=""): + def extract(self, progress=lambda x: None): """Extract the archive. Raise specific errors in case of failure. :param progress: Progress function, call this to update status - :param password password to use - :raises WrongPassword + :raises PasswordError :raises CRCError :raises ArchiveError :return: """ + self.setPassword(password) raise NotImplementedError -- cgit v1.2.3 From f71c1ef70a199e42e8a519364d9924e138ffd37c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 24 Dec 2014 15:48:08 +0100 Subject: [ExtractArchive] Remove empty directory --- module/plugins/internal/UnRar.py | 2 +- module/plugins/internal/UnZip.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 5633b31f7..b70bf3257 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -8,7 +8,7 @@ from os.path import basename, dirname, join from string import digits from subprocess import Popen, PIPE -from module.plugins.internal.AbstractExtractor import AbtractExtractor, PasswordError, ArchiveError, CRCError +from module.plugins.internal.AbstractExtractor import AbtractExtractor, ArchiveError, CRCError, PasswordError from module.utils import save_join, decode diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index b3d54cba0..875a0ec09 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -5,7 +5,7 @@ from __future__ import with_statement import sys import zipfile -from module.plugins.internal.AbstractExtractor import AbtractExtractor, PasswordError, ArchiveError, CRCError +from module.plugins.internal.AbstractExtractor import AbtractExtractor, ArchiveError, CRCError, PasswordError class UnZip(AbtractExtractor): -- cgit v1.2.3 From 4c63928557398891c30d3e2b7c962a07b3483315 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Dec 2014 04:18:41 +0100 Subject: Rename AbstractExtractor to Extractor --- module/plugins/internal/AbstractExtractor.py | 141 --------------------------- module/plugins/internal/Extractor.py | 140 ++++++++++++++++++++++++++ module/plugins/internal/UnRar.py | 6 +- module/plugins/internal/UnZip.py | 6 +- 4 files changed, 146 insertions(+), 147 deletions(-) delete mode 100644 module/plugins/internal/AbstractExtractor.py create mode 100644 module/plugins/internal/Extractor.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/AbstractExtractor.py b/module/plugins/internal/AbstractExtractor.py deleted file mode 100644 index 310897d4e..000000000 --- a/module/plugins/internal/AbstractExtractor.py +++ /dev/null @@ -1,141 +0,0 @@ -# -*- coding: utf-8 -*- - -class ArchiveError(Exception): - pass - - -class CRCError(Exception): - pass - - -class PasswordError(Exception): - pass - - -class AbtractExtractor: - __name__ = "AbtractExtractor" - __version__ = "0.12" - - __description__ = """Abtract extractor plugin""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "ranan@pyload.org"), - ("Walter Purcaro", "vuolter@gmail.com")] - - - EXTENSIONS = [] - - - @classmethod - def checkDeps(cls): - """ Check if system statisfy dependencies - :return: boolean - """ - return True - - - @classmethod - def isArchive(cls, file): - raise NotImplementedError - - - @classmethod - def getTargets(cls, files_ids): - """ Filter suited targets from list of filename id tuple list - :param files_ids: List of filepathes - :return: List of targets, id tuple list - """ - targets = [] - - for file, id in files_ids: - if cls.isArchive(file): - targets.append((file, id)) - - return targets - - - def __init__(self, m, file, out, password, fullpath, overwrite, excludefiles, renice, delete, keepbroken): - """Initialize extractor for specific file - - :param m: ExtractArchive Hook plugin - :param file: Absolute filepath - :param out: Absolute path to destination directory - :param fullpath: extract to fullpath - :param overwrite: Overwrite existing archives - :param renice: Renice value - """ - self.m = m - self.file = file - self.out = out - self.password = password - self.fullpath = fullpath - self.overwrite = overwrite - self.excludefiles = excludefiles - self.renice = renice - self.delete = delete - self.keepbroken = keepbroken - self.files = [] #: Store extracted files here - - - def init(self): - """ Initialize additional data structures """ - pass - - - def verify(self): - """Check if password if needed. Raise ArchiveError if integrity is - questionable. - - :raises ArchiveError - """ - pass - - - def isPassword(self, password): - """ Check if the given password is/might be correct. - If it can not be decided at this point return true. - - :param password: - :return: boolean - """ - if isinstance(password, basestring): - return True - else: - return False - - - def setPassword(self, password): - if self.isPassword(password): - self.password = password - return True - else: - return False - - - def repair(self): - return False - - - def extract(self, progress=lambda x: None): - """Extract the archive. Raise specific errors in case of failure. - - :param progress: Progress function, call this to update status - :raises PasswordError - :raises CRCError - :raises ArchiveError - :return: - """ - self.setPassword(password) - raise NotImplementedError - - - def getDeleteFiles(self): - """Return list of files to delete, do *not* delete them here. - - :return: List with paths of files to delete - """ - raise NotImplementedError - - - def getExtractedFiles(self): - """Populate self.files at some point while extracting""" - return self.files diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py new file mode 100644 index 000000000..0b2462dac --- /dev/null +++ b/module/plugins/internal/Extractor.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- + +class ArchiveError(Exception): + pass + + +class CRCError(Exception): + pass + + +class PasswordError(Exception): + pass + + +class Extractor: + __name__ = "Extractor" + __version__ = "0.13" + + __description__ = """Base extractor plugin""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "ranan@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + EXTENSIONS = [] + + + @classmethod + def checkDeps(cls): + """ Check if system statisfy dependencies + :return: boolean + """ + return True + + + @classmethod + def isArchive(cls, file): + raise NotImplementedError + + + @classmethod + def getTargets(cls, files_ids): + """ Filter suited targets from list of filename id tuple list + :param files_ids: List of filepathes + :return: List of targets, id tuple list + """ + targets = [] + + for file, id in files_ids: + if cls.isArchive(file): + targets.append((file, id)) + + return targets + + + def __init__(self, m, file, out, password, fullpath, overwrite, excludefiles, renice, delete, keepbroken): + """Initialize extractor for specific file + + :param m: ExtractArchive Hook plugin + :param file: Absolute filepath + :param out: Absolute path to destination directory + :param fullpath: extract to fullpath + :param overwrite: Overwrite existing archives + :param renice: Renice value + """ + self.m = m + self.file = file + self.out = out + self.password = password + self.fullpath = fullpath + self.overwrite = overwrite + self.excludefiles = excludefiles + self.renice = renice + self.delete = delete + self.keepbroken = keepbroken + self.files = [] #: Store extracted files here + + + def init(self): + """ Initialize additional data structures """ + pass + + + def verify(self): + """Check if password if needed. Raise ArchiveError if integrity is + questionable. + + :raises ArchiveError + """ + pass + + + def isPassword(self, password): + """ Check if the given password is/might be correct. + If it can not be decided at this point return true. + + :param password: + :return: boolean + """ + if isinstance(password, basestring): + return True + else: + return False + + + def setPassword(self, password): + if self.isPassword(password): + self.password = password + return True + else: + return False + + + def repair(self): + return False + + + def extract(self, progress=lambda x: None): + """Extract the archive. Raise specific errors in case of failure. + + :param progress: Progress function, call this to update status + :raises PasswordError + :raises CRCError + :raises ArchiveError + :return: + """ + raise NotImplementedError + + + def getDeleteFiles(self): + """Return list of files to delete, do *not* delete them here. + + :return: List with paths of files to delete + """ + raise NotImplementedError + + + def getExtractedFiles(self): + """Populate self.files at some point while extracting""" + return self.files diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index b70bf3257..572fe95b9 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -8,7 +8,7 @@ from os.path import basename, dirname, join from string import digits from subprocess import Popen, PIPE -from module.plugins.internal.AbstractExtractor import AbtractExtractor, ArchiveError, CRCError, PasswordError +from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError from module.utils import save_join, decode @@ -20,9 +20,9 @@ def renice(pid, value): print "Renice failed" -class UnRar(AbtractExtractor): +class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.00" + __version__ = "1.01" __description__ = """Rar extractor plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index 875a0ec09..5ec56cbdf 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -5,12 +5,12 @@ from __future__ import with_statement import sys import zipfile -from module.plugins.internal.AbstractExtractor import AbtractExtractor, ArchiveError, CRCError, PasswordError +from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -class UnZip(AbtractExtractor): +class UnZip(Extractor): __name__ = "UnZip" - __version__ = "1.00" + __version__ = "1.01" __description__ = """Zip extractor plugin""" __license__ = "GPLv3" -- cgit v1.2.3 From f37514e315e076379ab73539ffa36e48671976cc Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Dec 2014 04:19:57 +0100 Subject: [MultiHoster] Improve --- module/plugins/internal/MultiHoster.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index e5c28d034..6bfff171b 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -2,13 +2,13 @@ import re -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, replace_patterns, set_cookies class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.24" + __version__ = "0.25" __pattern__ = r'^unmatchable$' @@ -17,7 +17,7 @@ class MultiHoster(SimpleHoster): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - DIRECT_LINK = True + LOGIN_ACCOUNT = True def setup(self): @@ -25,6 +25,28 @@ class MultiHoster(SimpleHoster): self.multiDL = self.premium + def prepare(self): + self.info = {} + self.link = "" #@TODO: Move to hoster class in 0.4.10 + self.directDL = False #@TODO: Move to hoster class in 0.4.10 + + if self.LOGIN_ACCOUNT and not self.account: + self.fail(_("Required account not found")) + + self.req.setOption("timeout", 120) + + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) + + if self.DIRECT_LINK is None: + self.directDL = self.__pattern__ != r'^unmatchable$' + else: + self.directDL = self.DIRECT_LINK + + self.pyfile.url = replace_patterns(self.pyfile.url, + self.FILE_URL_REPLACEMENTS if hasattr(self, "FILE_URL_REPLACEMENTS") else self.URL_REPLACEMENTS) #@TODO: Remove FILE_URL_REPLACEMENTS check in 0.4.10 + + def process(self, pyfile): self.prepare() @@ -35,7 +57,7 @@ class MultiHoster(SimpleHoster): if not self.link and not self.lastDownload: self.preload() - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + if self.premium and not self.CHECK_TRAFFIC or self.checkTrafficLeft(): self.logDebug("Handled as premium download") self.handlePremium() @@ -55,4 +77,4 @@ class MultiHoster(SimpleHoster): if self.premium: raise NotImplementedError else: - self.logError(_("Required account not found")) + self.fail(_("Required premium account not found")) -- cgit v1.2.3 From 6d3b96012ba64343586ba53041650d93493a335c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Dec 2014 04:23:45 +0100 Subject: [MultiHook] Overwrites account if not premium --- module/plugins/internal/MultiHook.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index dcf1c3383..caac6c258 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -9,7 +9,7 @@ from module.utils import remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.22" + __version__ = "0.23" __description__ = """Hook plugin for MultiHoster""" __license__ = "GPLv3" @@ -130,14 +130,14 @@ class MultiHook(Hook): old_supported = [hoster for hoster in old_supported if hoster not in self.supported] if old_supported: - self.logDebug("UNLOAD", ", ".join(old_supported)) + self.logDebug("Unload: %s" % ", ".join(old_supported)) for hoster in old_supported: self.unloadHoster(hoster) def overridePlugins(self): pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys()) - accountList = [name.lower() for name, data in self.core.accountManager.accounts.iteritems() if data] + accountList = [account.type.lower() for account in self.core.api.getAccounts(False) if account.valid and account.premium] excludedList = [] for hoster in self.getHosterCached(): @@ -159,26 +159,26 @@ class MultiHook(Hook): klass = getattr(module, self.__name__) # inject plugin plugin - self.logDebug("Overwritten Hosters", ", ".join(sorted(self.supported))) + self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) for hoster in self.supported: hdict = self.core.pluginManager.hosterPlugins[hoster] hdict['new_module'] = module hdict['new_name'] = self.__name__ if excludedList: - self.logInfo(_("The following hosters were not overwritten - account exists"), ", ".join(sorted(excludedList))) + self.logInfo(_("Hosters not overwritten: %s" % ", ".join(sorted(excludedList)))) if self.new_supported: hosters = sorted(self.new_supported) - self.logDebug("New Hosters", ", ".join(hosters)) + self.logDebug("New Hosters: %s" % ", ".join(hosters)) # create new regexp regexp = r'.*(%s).*' % "|".join([x.replace(".", "\.") for x in hosters]) if hasattr(klass, "__pattern__") and isinstance(klass.__pattern__, basestring) and '://' in klass.__pattern__: regexp = r'%s|%s' % (klass.__pattern__, regexp) - self.logDebug("Regexp", regexp) + self.logDebug("Regexp: %s" % regexp) hdict = self.core.pluginManager.hosterPlugins[self.__name__] hdict['pattern'] = regexp -- cgit v1.2.3 From 873f1bc17733148163829e23a456a6e724c486e3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Dec 2014 04:26:20 +0100 Subject: [SimpleHoster] Improve --- module/plugins/internal/SimpleHoster.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 6726726e1..56c25e0d9 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -160,7 +160,7 @@ def _isDirectLink(self, url, resumable=False): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.79" + __version__ = "0.80" __pattern__ = r'^unmatchable$' @@ -381,7 +381,7 @@ class SimpleHoster(Hoster): if self.html is None: self.fail(_("No html retrieved")) - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + if self.premium and not self.CHECK_TRAFFIC or self.checkTrafficLeft(): self.logDebug("Handled as premium download") self.handlePremium() @@ -405,7 +405,11 @@ class SimpleHoster(Hoster): self.retry(10, reason=_("Wrong captcha")) elif not self.lastDownload or not exists(fs_encode(self.lastDownload)): - self.fail(_("No file downloaded")) + errmsg = _("No file downloaded") + if 'error' in self.info: + self.fail(errmsg, self.info['error']) + else: + self.fail(errmsg) else: rules = {'empty file': re.compile(r"^$")} -- cgit v1.2.3 From 136f63dc39603814b215606f888fb2e639021277 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Dec 2014 04:27:41 +0100 Subject: Spare code fixes --- module/plugins/internal/SimpleCrypter.py | 10 +++++----- module/plugins/internal/XFSHoster.py | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index e0dd10585..6596828b3 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -12,7 +12,7 @@ from module.utils import fixup class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.33" + __version__ = "0.34" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -82,14 +82,14 @@ class SimpleCrypter(Crypter, SimpleHoster): def prepare(self): - if self.LOGIN_ACCOUNT and not self.account: - self.fail(_("Required account not found")) + self.info = {} + self.links = [] #@TODO: Move to hoster class in 0.4.10 if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) - self.info = {} - self.links = [] + if self.LOGIN_ACCOUNT and not self.account: + self.fail(_("Required account not found")) self.req.setOption("timeout", 120) diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index dd2dd2527..7c6ae5725 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.31" + __version__ = "0.32" __pattern__ = r'^unmatchable$' @@ -32,7 +32,7 @@ class XFSHoster(SimpleHoster): TEXT_ENCODING = False COOKIES = [(HOSTER_DOMAIN, "lang", "english")] - CHECK_DIRECT_LINK = None + DIRECT_LINK = None MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... NAME_PATTERN = r'(Filename[ ]*:[ ]*()?|name="fname"[ ]+value="|<[\w^_]+ class="(file)?name">)\s*(?P.+?)(\s*<|")' @@ -45,7 +45,7 @@ class XFSHoster(SimpleHoster): PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only' ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter>|>Error|>\(ERROR:)(?:\s*<.+?>\s*)*(.+?)(?:["\']|<|\))' - LEECH_LINK_PATTERN = r'

Download Link

\s*]*>([^<]+)' + LINK_LEECH_PATTERN = r'

Download Link

\s*]*>([^<]+)' LINK_PATTERN = None #: final download url pattern CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' @@ -58,7 +58,7 @@ class XFSHoster(SimpleHoster): def setup(self): - self.chunkLimit = 1 + self.chunkLimit = 1 self.resumeDownload = self.multiDL = self.premium @@ -80,7 +80,7 @@ class XFSHoster(SimpleHoster): super(XFSHoster, self).prepare() - if self.CHECK_DIRECT_LINK is None: + if self.DIRECT_LINK is None: self.directDL = bool(self.premium) @@ -189,9 +189,9 @@ class XFSHoster(SimpleHoster): self.fail(stmsg) #get easybytez.com link for uploaded file - m = re.search(self.LEECH_LINK_PATTERN, self.html) + m = re.search(self.LINK_LEECH_PATTERN, self.html) if m is None: - self.error(_("LEECH_LINK_PATTERN not found")) + self.error(_("LINK_LEECH_PATTERN not found")) header = self.load(m.group(1), just_header=True, decode=True) -- cgit v1.2.3 From c7187f2142c6e06be961bb3be45ea3b61ce31651 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Dec 2014 17:24:41 +0100 Subject: [SimpleHoster] getInfo handle direct link error --- module/plugins/internal/MultiHoster.py | 4 ++-- module/plugins/internal/SimpleHoster.py | 23 +++++++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 6bfff171b..9d218c3b2 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -8,7 +8,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.25" + __version__ = "0.26" __pattern__ = r'^unmatchable$' @@ -57,7 +57,7 @@ class MultiHoster(SimpleHoster): if not self.link and not self.lastDownload: self.preload() - if self.premium and not self.CHECK_TRAFFIC or self.checkTrafficLeft(): + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): self.logDebug("Handled as premium download") self.handlePremium() diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 56c25e0d9..89d2e629c 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -160,7 +160,7 @@ def _isDirectLink(self, url, resumable=False): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.80" + __version__ = "0.81" __pattern__ = r'^unmatchable$' @@ -233,7 +233,13 @@ class SimpleHoster(Hoster): @classmethod def getInfo(cls, url="", html=""): - info = {'name': urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url} + info = {'name': urlparse(unquote(url)).path.split('/')[-1] or _("Unknown"), 'size': 0, 'status': 3, 'url': url} + online = False + + try: + info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here, please save api stuff to info['api'] + except Exception: + pass if not html: try: @@ -242,6 +248,11 @@ class SimpleHoster(Hoster): info['status'] = 1 raise + if _isDirectLink(url): + info['error'] = "direct link" + info['status'] = 2 + raise + try: html = getURL(url, cookies=cls.COOKIES, decode=not cls.TEXT_ENCODING) @@ -261,8 +272,6 @@ class SimpleHoster(Hoster): except: return info - online = False - if hasattr(cls, "OFFLINE_PATTERN") and re.search(cls.OFFLINE_PATTERN, html): info['status'] = 1 @@ -273,9 +282,7 @@ class SimpleHoster(Hoster): info['status'] = 6 else: - try: - info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: pattern groups will be saved here, please save api stuff to info['api'] - except: + if not 'pattern' in info: info['pattern'] = {} for pattern in ("FILE_INFO_PATTERN", "INFO_PATTERN", @@ -381,7 +388,7 @@ class SimpleHoster(Hoster): if self.html is None: self.fail(_("No html retrieved")) - if self.premium and not self.CHECK_TRAFFIC or self.checkTrafficLeft(): + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): self.logDebug("Handled as premium download") self.handlePremium() -- cgit v1.2.3 From ff42cec46274b0f2019272424c1a57ec4b413507 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 26 Dec 2014 17:58:54 +0100 Subject: [PremiumTo] Fixup --- module/plugins/internal/MultiHook.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index caac6c258..78de1ed0a 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -47,6 +47,7 @@ class MultiHook(Hook): """getConfig with default value - sublass may not implements all config options""" try: return self.getConf(option) + except KeyError: return default -- cgit v1.2.3 From 4f287a307dd61afbbf1029065dcd6d1b9c304a15 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 27 Dec 2014 13:39:25 +0100 Subject: New plugin: SimpleDereferer --- module/plugins/internal/SimpleDereferer.py | 96 ++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 module/plugins/internal/SimpleDereferer.py (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleDereferer.py b/module/plugins/internal/SimpleDereferer.py new file mode 100644 index 000000000..0ad1098f4 --- /dev/null +++ b/module/plugins/internal/SimpleDereferer.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- + +import re + +from urllib import unquote + +from module.plugins.Crypter import Crypter +from module.plugins.internal.SimpleHoster import _isDirectLink, set_cookies + + +class SimpleDereferer(Crypter): + __name__ = "SimpleDereferer" + __type__ = "crypter" + __version__ = "0.01" + + __pattern__ = r'^unmatchable$' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Simple dereferer plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + """ + Following patterns should be defined by each crypter: + + LINK_PATTERN: Regex to catch the redirect url in group(1) + example: LINK_PATTERN = r'
Date: Sat, 27 Dec 2014 13:44:36 +0100 Subject: [SimpleHoster] Improve _isDirectLink + move here secondsToMidnight routine + spare code cosmetics --- module/plugins/internal/SimpleHoster.py | 55 +++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 16 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 89d2e629c..d9732d063 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -131,7 +131,7 @@ def _isDirectLink(self, url, resumable=False): for i in xrange(5 if resumable else 1): header = self.load(url, ref=True, cookies=True, just_header=True, decode=True) - if 'content-disposition' in header: + if 'content-disposition' in header or 'content-length' in header: link = url elif 'location' in header and header['location']: @@ -150,6 +150,9 @@ def _isDirectLink(self, url, resumable=False): self.logDebug("Redirect #%d to: %s" % (++i, location)) continue + elif 'content-type' in header and header['content-type' ] and "html" not in header['content-type']: + link = url + break else: self.logError(_("Too many redirects")) @@ -157,10 +160,28 @@ def _isDirectLink(self, url, resumable=False): return link +def secondsToMidnight(gmt=0): + now = datetime.utcnow() + timedelta(hours=gmt) + + if now.hour is 0 and now.minute < 10: + midnight = now + else: + midnight = now + timedelta(days=1) + + td = midnight.replace(hour=0, minute=10, second=0, microsecond=0) - now + + if hasattr(td, 'total_seconds'): + res = td.total_seconds() + else: #@NOTE: work-around for python 2.5 and 2.6 missing timedelta.total_seconds + res = (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 + + return int(res) + + class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.81" + __version__ = "0.82" __pattern__ = r'^unmatchable$' @@ -185,10 +206,10 @@ class SimpleHoster(Hoster): HASHSUM_PATTERN: (optional) Hash code and type of the file example: HASHSUM_PATTERN = r'(?Phash_code) (?PMD5)' - OFFLINE_PATTERN: (optional) Check if the file is yet available online + OFFLINE_PATTERN: (optional) Check if the page is unreachable example: OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN: (optional) Check if the file is temporarily offline + TEMP_OFFLINE_PATTERN: (optional) Check if the page is temporarily unreachable example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' @@ -217,11 +238,11 @@ class SimpleHoster(Hoster): SIZE_REPLACEMENTS = [] URL_REPLACEMENTS = [] - TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct - COOKIES = True #: or False or list of tuples [(domain, name, value)] - CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - DIRECT_LINK = None #: Set to True to looking for direct link (as defined in handleDirect method), set to None to do it if self.account is True else False - MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) + TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct + COOKIES = True #: or False or list of tuples [(domain, name, value)] + CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account + DIRECT_LINK = None #: Set to True to looking for direct link (as defined in handleDirect method), set to None to do it if self.account is True else False + MULTI_HOSTER = False #: Set to True to leech other hoster link (as defined in handleMulti method) @classmethod @@ -451,7 +472,10 @@ class SimpleHoster(Hoster): self.info.pop('error', None) - def checkStatus(self): + def checkStatus(self, getinfo=True): + if getinfo: + self.updateInfo(self.getInfo(self.pyfile.url, self.html)) + status = self.info['status'] if status is 1: @@ -465,7 +489,10 @@ class SimpleHoster(Hoster): "File info: %s" % self.info) - def checkNameSize(self): + def checkNameSize(self, getinfo=True): + if getinfo: + self.updateInfo(self.getInfo(self.pyfile.url, self.html)) + name = self.info['name'] size = self.info['size'] url = self.info['url'] @@ -485,17 +512,13 @@ class SimpleHoster(Hoster): def checkInfo(self): - self.updateInfo(self.getInfo(self.pyfile.url, self.html)) - self.checkNameSize() if self.html: self.checkErrors() - self.updateInfo(self.getInfo(self.pyfile.url, self.html)) - self.checkNameSize() - self.checkStatus() + self.checkStatus(getinfo=False) #: Deprecated -- cgit v1.2.3 From b6a2bd53628bd2824bac1dafc7b4eafdf3c815dd Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 27 Dec 2014 13:48:03 +0100 Subject: Update plugins after SimpleHoster changes --- module/plugins/internal/SimpleCrypter.py | 15 +++++++++------ module/plugins/internal/XFSHoster.py | 17 ++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 6596828b3..428826456 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -12,7 +12,7 @@ from module.utils import fixup class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.34" + __version__ = "0.35" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -28,16 +28,16 @@ class SimpleCrypter(Crypter, SimpleHoster): """ Following patterns should be defined by each crypter: - LINK_PATTERN: group(1) must be a download link or a regex to catch more links + LINK_PATTERN: Download link or regex to catch links in group(1) example: LINK_PATTERN = r'
Files of: (?P[^<]+) folder' - OFFLINE_PATTERN: (optional) Checks if the file is yet available online + OFFLINE_PATTERN: (optional) Checks if the page is unreachable example: OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN: (optional) Checks if the file is temporarily offline + TEMP_OFFLINE_PATTERN: (optional) Checks if the page is temporarily unreachable example: TEMP_OFFLINE_PATTERN = r'Server maintainance' @@ -120,7 +120,10 @@ class SimpleCrypter(Crypter, SimpleHoster): self.packages = [(self.info['name'], self.links, self.info['folder'])] - def checkNameSize(self): + def checkNameSize(self, getinfo=True): + if getinfo: + self.updateInfo(self.getInfo(self.pyfile.url, self.html)) + name = self.info['name'] url = self.info['url'] diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 7c6ae5725..f2168d0c7 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -7,16 +7,15 @@ from time import sleep from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME -from module.plugins.hoster.UnrestrictLi import secondsToMidnight from module.plugins.internal.CaptchaService import ReCaptcha, SolveMedia -from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo +from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, secondsToMidnight from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.32" + __version__ = "0.33" __pattern__ = r'^unmatchable$' @@ -30,10 +29,10 @@ class XFSHoster(SimpleHoster): HOSTER_DOMAIN = None HOSTER_NAME = None - TEXT_ENCODING = False - COOKIES = [(HOSTER_DOMAIN, "lang", "english")] - DIRECT_LINK = None - MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... + TEXT_ENCODING = False + COOKIES = [(HOSTER_DOMAIN, "lang", "english")] + DIRECT_LINK = None + MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... NAME_PATTERN = r'(Filename[ ]*:[ ]*()?|name="fname"[ ]+value="|<[\w^_]+ class="(file)?name">)\s*(?P.+?)(\s*<|")' SIZE_PATTERN = r'(Size[ ]*:[ ]*()?|File:.*>|\s*\(|<[\w^_]+ class="size">)\s*(?P[\d.,]+)\s*(?P[\w^_]+)' @@ -226,10 +225,10 @@ class XFSHoster(SimpleHoster): elif 'limit' in self.errmsg: if 'days' in self.errmsg: - delay = secondsToMidnight(gmt=2) + delay = secondsToMidnight(gmt=2) retries = 3 else: - delay = 1 * 60 * 60 + delay = 1 * 60 * 60 retries = 24 self.wantReconnect = True -- cgit v1.2.3 From b72be64cc16660e3ba877d97dbc0e717bb030666 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 27 Dec 2014 21:14:40 +0100 Subject: [MultiHook] Crypter support --- module/plugins/internal/MultiHook.py | 150 +++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 67 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index 78de1ed0a..cf251bacf 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -9,16 +9,22 @@ from module.utils import remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.23" + __version__ = "0.25" - __description__ = """Hook plugin for MultiHoster""" + __config__ = [("mode" , "all;listed;unlisted", "Use for plugins (if supported)" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed", "bool" , "Revert to standard download if download fails", False), + ("interval" , "int" , "Reload interval in hours (0 to disable)" , 12 )] + + __description__ = """Hook plugin for multi hoster/crypter""" __license__ = "GPLv3" - __authors__ = [("pyLoad Team", "admin@pyload.org")] + __authors__ = [("pyLoad Team", "admin@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] - interval = 12 * 60 * 60 #: reload hosters every 12h + MIN_INTERVAL = 12 * 60 * 60 #: reload plugins every 12h - HOSTER_REPLACEMENTS = [("1fichier.com" , "onefichier.com"), + PLUGIN_REPLACEMENTS = [("1fichier.com" , "onefichier.com"), ("2shared.com" , "twoshared.com" ), ("4shared.com" , "fourshared.com"), ("cloudnator.com" , "shragle.com" ), @@ -34,15 +40,26 @@ class MultiHook(Hook): ("sharerapid.cz" , "multishare.cz" ), ("ul.to" , "uploaded.to" ), ("uploaded.net" , "uploaded.to" )] - HOSTER_EXCLUDED = [] def setup(self): - self.hosters = [] + self.type = self.core.pluginManager.findPlugin(self.__name__)[1] or "hoster" + self.plugins = [] self.supported = [] self.new_supported = [] + def getURL(self, *args, **kwargs): #@TODO: Remove in 0.4.10 + """ see HTTPRequest for argument list """ + h = pyreq.getHTTPRequest(timeout=120) + try: + rep = h.load(*args, **kwargs) + finally: + h.close() + + return rep + + def getConfig(self, option, default=''): """getConfig with default value - sublass may not implements all config options""" try: @@ -52,36 +69,37 @@ class MultiHook(Hook): return default - def getHosterCached(self): - if not self.hosters: + def pluginCached(self): + if not self.plugins: try: - hosterSet = self.toHosterSet(self.getHoster()) - set(self.HOSTER_EXCLUDED) + pluginset = self.pluginSet(self.getHosters() if self.type is "hoster" else self.getCrypters()) except Exception, e: self.logError(e) return [] try: - configMode = self.getConfig('hosterListMode', 'all') - if configMode in ("listed", "unlisted"): - configSet = self.toHosterSet(self.getConfig('hosterList', '').replace('|', ',').replace(';', ',').split(',')) + configmode = self.getConfig("mode", 'all') + if configmode in ("listed", "unlisted"): + list = self.getConfig("pluginlist", '').replace('|', ',').replace(';', ',').split(',') + configset = self.pluginSet(list) - if configMode == "listed": - hosterSet &= configSet + if configmode == "listed": + pluginset &= configset else: - hosterSet -= configSet + pluginset -= configset except Exception, e: self.logError(e) - self.hosters = list(hosterSet) + self.plugins = list(pluginset) - return self.hosters + return self.plugins - def toHosterSet(self, hosters): + def pluginSet(self, hosters): hosters = set((str(x).strip().lower() for x in hosters)) - for rep in self.HOSTER_REPLACEMENTS: + for rep in self.PLUGIN_REPLACEMENTS: if rep[0] in hosters: hosters.remove(rep[0]) hosters.add(rep[1]) @@ -90,7 +108,7 @@ class MultiHook(Hook): return hosters - def getHoster(self): + def getHosters(self): """Load list of supported hoster :return: List of domain names @@ -98,96 +116,93 @@ class MultiHook(Hook): raise NotImplementedError - def coreReady(self): - if self.cb: - self.core.scheduler.removeJob(self.cb) - - self.setConfig("activated", True) #: config not in sync after plugin reload - - cfg_interval = self.getConfig("interval", None) #: reload interval in hours - if cfg_interval is not None: - self.interval = cfg_interval * 60 * 60 - - if self.interval: - self._periodical() - else: - self.periodical() - + def getCrypters(self): + """Load list of supported crypters - def initPeriodical(self): - pass + :return: List of domain names + """ + raise NotImplementedError def periodical(self): - """reload hoster list periodically""" - self.logInfo(_("Reloading supported hoster list")) + """reload plugin list periodically""" + self.interval = max(self.getConfig("interval", 0), self.MIN_INTERVAL) + + self.logInfo(_("Reloading supported %s list") % self.type) old_supported = self.supported self.supported = [] self.new_supported = [] - self.hosters = [] + self.plugins = [] self.overridePlugins() - old_supported = [hoster for hoster in old_supported if hoster not in self.supported] + old_supported = [plugin for plugin in old_supported if plugin not in self.supported] + if old_supported: self.logDebug("Unload: %s" % ", ".join(old_supported)) - for hoster in old_supported: - self.unloadHoster(hoster) + for plugin in old_supported: + self.unloadPlugin(plugin) def overridePlugins(self): - pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys()) - accountList = [account.type.lower() for account in self.core.api.getAccounts(False) if account.valid and account.premium] excludedList = [] - for hoster in self.getHosterCached(): - name = remove_chars(hoster, "-.") + if self.type is "hoster": + pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys()) + accountList = [account.type.lower() for account in self.core.api.getAccounts(False) if account.valid and account.premium] + else: + pluginMap = {} + accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.core.pluginManager.crypterPlugins.iterkeys()] + + for plugin in self.pluginCached(): + name = remove_chars(plugin, "-.") if name in accountList: - excludedList.append(hoster) + excludedList.append(plugin) else: if name in pluginMap: self.supported.append(pluginMap[name]) else: - self.new_supported.append(hoster) + self.new_supported.append(plugin) if not self.supported and not self.new_supported: - self.logError(_("No Hoster loaded")) + self.logError(_("No %s loaded") % self.type) return module = self.core.pluginManager.getPlugin(self.__name__) klass = getattr(module, self.__name__) # inject plugin plugin - self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) - for hoster in self.supported: - hdict = self.core.pluginManager.hosterPlugins[hoster] + self.logDebug("Overwritten %ss: %s" % (self.type, ", ".join(sorted(self.supported)))) + + for plugin in self.supported: + hdict = self.core.pluginManager.plugins[self.type][plugin] hdict['new_module'] = module hdict['new_name'] = self.__name__ if excludedList: - self.logInfo(_("Hosters not overwritten: %s" % ", ".join(sorted(excludedList)))) + self.logInfo(_("%ss not overwritten: %s") % (self.type.capitalize(), ", ".join(sorted(excludedList)))) if self.new_supported: - hosters = sorted(self.new_supported) + plugins = sorted(self.new_supported) - self.logDebug("New Hosters: %s" % ", ".join(hosters)) + self.logDebug("New %ss: %s" % (self.type, ", ".join(plugins))) # create new regexp - regexp = r'.*(%s).*' % "|".join([x.replace(".", "\.") for x in hosters]) + regexp = r'.*(%s).*' % "|".join([x.replace(".", "\.") for x in plugins]) if hasattr(klass, "__pattern__") and isinstance(klass.__pattern__, basestring) and '://' in klass.__pattern__: regexp = r'%s|%s' % (klass.__pattern__, regexp) self.logDebug("Regexp: %s" % regexp) - hdict = self.core.pluginManager.hosterPlugins[self.__name__] + hdict = self.core.pluginManager.plugins[self.type][self.__name__] hdict['pattern'] = regexp hdict['re'] = re.compile(regexp) - def unloadHoster(self, hoster): - hdict = self.core.pluginManager.hosterPlugins[hoster] + def unloadPlugin(self, plugin): + hdict = self.core.pluginManager.plugins[self.type][plugin] if "module" in hdict: del hdict['module'] @@ -198,21 +213,22 @@ class MultiHook(Hook): def unload(self): """Remove override for all hosters. Scheduler job is removed by hookmanager""" - for hoster in self.supported: - self.unloadHoster(hoster) + for plugin in self.supported: + self.unloadPlugin(plugin) # reset pattern klass = getattr(self.core.pluginManager.getPlugin(self.__name__), self.__name__) - hdict = self.core.pluginManager.hosterPlugins[self.__name__] + hdict = self.core.pluginManager.plugins[self.type][self.__name__] + hdict['pattern'] = getattr(klass, "__pattern__", r'^unmatchable$') hdict['re'] = re.compile(hdict['pattern']) def downloadFailed(self, pyfile): """remove plugin override if download fails but not if file is offline/temp.offline""" - if pyfile.hasStatus("failed") and self.getConfig("unloadFailing", True): - hdict = self.core.pluginManager.hosterPlugins[pyfile.pluginname] + if self.type is "hoster" and pyfile.hasStatus("failed") and self.getConfig("revertfailed", True): + hdict = self.core.pluginManager.plugins[self.type][pyfile.pluginname] if "new_name" in hdict and hdict['new_name'] == self.__name__: self.logDebug("Unload MultiHook", pyfile.pluginname, hdict) - self.unloadHoster(pyfile.pluginname) + self.unloadPlugin(pyfile.pluginname) pyfile.setStatus("queued") -- cgit v1.2.3 From 4882503efe1fce2f123723766f672456e5ccc221 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 27 Dec 2014 21:23:33 +0100 Subject: [SimpleCrypter][SimpleDereferer] Fail if no link grabbed --- module/plugins/internal/SimpleCrypter.py | 5 ++++- module/plugins/internal/SimpleDereferer.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 428826456..51ba07eb7 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -12,7 +12,7 @@ from module.utils import fixup class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.35" + __version__ = "0.36" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -119,6 +119,9 @@ class SimpleCrypter(Crypter, SimpleHoster): if self.links: self.packages = [(self.info['name'], self.links, self.info['folder'])] + elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 + self.fail("No link grabbed") + def checkNameSize(self, getinfo=True): if getinfo: diff --git a/module/plugins/internal/SimpleDereferer.py b/module/plugins/internal/SimpleDereferer.py index 0ad1098f4..04d63658e 100644 --- a/module/plugins/internal/SimpleDereferer.py +++ b/module/plugins/internal/SimpleDereferer.py @@ -11,7 +11,7 @@ from module.plugins.internal.SimpleHoster import _isDirectLink, set_cookies class SimpleDereferer(Crypter): __name__ = "SimpleDereferer" __type__ = "crypter" - __version__ = "0.01" + __version__ = "0.02" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), @@ -65,6 +65,9 @@ class SimpleDereferer(Crypter): if link.strip(): self.urls = [link.strip()] #@TODO: Remove `.strip()` in 0.4.10 + elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 + self.fail("No link grabbed") + def prepare(self): self.req.setOption("timeout", 120) -- cgit v1.2.3 From 1c443921febd75050968bb37e7fc88e361c487a9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 27 Dec 2014 21:27:00 +0100 Subject: [XFSHoster] Code cosmetics --- module/plugins/internal/XFSHoster.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index f2168d0c7..4555e7b0f 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -73,9 +73,8 @@ class XFSHoster(SimpleHoster): pattern = r'(https?://(www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]' self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.') - self.captcha = None - self.errmsg = None - self.passwords = self.getPassword().splitlines() + self.captcha = None + self.errmsg = None super(XFSHoster, self).prepare() @@ -271,8 +270,9 @@ class XFSHoster(SimpleHoster): if 'op' in inputs: if "password" in inputs: - if self.passwords: - inputs['password'] = self.passwords.pop(0) + password = self.getPassword() + if password: + inputs['password'] = password else: self.fail(_("Missing password")) -- cgit v1.2.3 From 388c894716e670e89b88b0b05f99d902d0bc054d Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 28 Dec 2014 01:05:27 +0100 Subject: [MultiHoster] Filename recognition --- module/plugins/internal/MultiHoster.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index 9d218c3b2..8cdb2fbff 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -8,7 +8,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.26" + __version__ = "0.27" __pattern__ = r'^unmatchable$' @@ -54,13 +54,17 @@ class MultiHoster(SimpleHoster): self.logDebug("Looking for direct download link...") self.handleDirect() - if not self.link and not self.lastDownload: + if self.link: + self.pyfile.url = self.link + self.checkNameSize() + + elif not self.lastDownload: self.preload() + self.checkInfo() if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): self.logDebug("Handled as premium download") self.handlePremium() - else: self.logDebug("Handled as free download") self.handleFree() -- cgit v1.2.3 From 0c61cbb03493b15970a5ccb9a5e20b02b3884a4e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 28 Dec 2014 01:05:59 +0100 Subject: [SimpleHoster] Improve --- module/plugins/internal/SimpleCrypter.py | 4 ---- module/plugins/internal/SimpleHoster.py | 7 ++----- 2 files changed, 2 insertions(+), 9 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 51ba07eb7..566252245 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -103,10 +103,6 @@ class SimpleCrypter(Crypter, SimpleHoster): self.prepare() self.preload() - - if self.html is None: - self.fail(_("No html retrieved")) - self.checkInfo() self.links = self.getLinks() diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index d9732d063..5c6f8c720 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -181,7 +181,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.82" + __version__ = "0.83" __pattern__ = r'^unmatchable$' @@ -388,7 +388,7 @@ class SimpleHoster(Hoster): def process(self, pyfile): self.prepare() - self.checkInfo() + self.checkNameSize() if self.directDL: self.logDebug("Looking for direct download link...") @@ -406,9 +406,6 @@ class SimpleHoster(Hoster): self.preload() self.checkInfo() - if self.html is None: - self.fail(_("No html retrieved")) - if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): self.logDebug("Handled as premium download") self.handlePremium() -- cgit v1.2.3 From 98ae3d727b60d3f529b39a6c275e31732231bd91 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 28 Dec 2014 11:53:37 +0100 Subject: [SimpleHoster] Fix https://github.com/pyload/pyload/issues/983 --- module/plugins/internal/SimpleHoster.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 5c6f8c720..6075e3f67 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -181,7 +181,7 @@ def secondsToMidnight(gmt=0): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.83" + __version__ = "0.84" __pattern__ = r'^unmatchable$' @@ -444,7 +444,9 @@ class SimpleHoster(Hoster): check = self.checkDownload(rules) if check: #@TODO: Move to hoster in 0.4.10 - errmsg = check.strip().capitalize() + (" | " + self.lastCheck.strip() if self.lastCheck else "") + errmsg = check.strip().capitalize() + if self.lastCheck: + errmsg += " | " + self.lastCheck.group(0).strip() self.retry(10, 60, errmsg) -- cgit v1.2.3 From e4cf98e1f368eb846469d1acaf0efd015bf2a921 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 30 Dec 2014 15:43:11 +0100 Subject: [MultiHook] Fixup --- module/plugins/internal/MultiHook.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'module/plugins/internal') diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py index cf251bacf..8164a02fc 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiHook.py @@ -9,7 +9,7 @@ from module.utils import remove_chars class MultiHook(Hook): __name__ = "MultiHook" __type__ = "hook" - __version__ = "0.25" + __version__ = "0.26" __config__ = [("mode" , "all;listed;unlisted", "Use for plugins (if supported)" , "all"), ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), @@ -72,7 +72,7 @@ class MultiHook(Hook): def pluginCached(self): if not self.plugins: try: - pluginset = self.pluginSet(self.getHosters() if self.type is "hoster" else self.getCrypters()) + pluginset = self.pluginSet(self.getHosters() if self.type == "hoster" else self.getCrypters()) except Exception, e: self.logError(e) return [] @@ -80,8 +80,8 @@ class MultiHook(Hook): try: configmode = self.getConfig("mode", 'all') if configmode in ("listed", "unlisted"): - list = self.getConfig("pluginlist", '').replace('|', ',').replace(';', ',').split(',') - configset = self.pluginSet(list) + pluginlist = self.getConfig("pluginlist", '').replace('|', ',').replace(';', ',').split(',') + configset = self.pluginSet(pluginlist) if configmode == "listed": pluginset &= configset @@ -96,16 +96,17 @@ class MultiHook(Hook): return self.plugins - def pluginSet(self, hosters): - hosters = set((str(x).strip().lower() for x in hosters)) + def pluginSet(self, plugins): + plugins = set((str(x).strip().lower() for x in plugins)) for rep in self.PLUGIN_REPLACEMENTS: - if rep[0] in hosters: - hosters.remove(rep[0]) - hosters.add(rep[1]) + if rep[0] in plugins: + plugins.remove(rep[0]) + plugins.add(rep[1]) - hosters.discard('') - return hosters + plugins.discard('') + + return plugins def getHosters(self): @@ -148,7 +149,7 @@ class MultiHook(Hook): def overridePlugins(self): excludedList = [] - if self.type is "hoster": + if self.type == "hoster": pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys()) accountList = [account.type.lower() for account in self.core.api.getAccounts(False) if account.valid and account.premium] else: @@ -212,7 +213,7 @@ class MultiHook(Hook): def unload(self): - """Remove override for all hosters. Scheduler job is removed by hookmanager""" + """Remove override for all plugins. Scheduler job is removed by hookmanager""" for plugin in self.supported: self.unloadPlugin(plugin) @@ -226,7 +227,7 @@ class MultiHook(Hook): def downloadFailed(self, pyfile): """remove plugin override if download fails but not if file is offline/temp.offline""" - if self.type is "hoster" and pyfile.hasStatus("failed") and self.getConfig("revertfailed", True): + if pyfile.hasStatus("failed") and self.getConfig("revertfailed", True): hdict = self.core.pluginManager.plugins[self.type][pyfile.pluginname] if "new_name" in hdict and hdict['new_name'] == self.__name__: self.logDebug("Unload MultiHook", pyfile.pluginname, hdict) -- cgit v1.2.3