diff options
author | Walter Purcaro <vuolter@users.noreply.github.com> | 2015-09-29 21:43:22 +0200 |
---|---|---|
committer | Walter Purcaro <vuolter@users.noreply.github.com> | 2015-09-29 21:43:22 +0200 |
commit | 8af39bfa2c409d4fd07d430485117bf60fdcc935 (patch) | |
tree | 9340941b94be82a9aec40544eef7e8f1aeb766e9 | |
parent | Account class completely rewritten + plugins updated (diff) | |
download | pyload-8af39bfa2c409d4fd07d430485117bf60fdcc935.tar.xz |
Update internal plugins
-rw-r--r-- | module/plugins/internal/Addon.py | 1 | ||||
-rw-r--r-- | module/plugins/internal/Container.py | 1 | ||||
-rw-r--r-- | module/plugins/internal/Crypter.py | 6 | ||||
-rw-r--r-- | module/plugins/internal/Extractor.py | 3 | ||||
-rw-r--r-- | module/plugins/internal/Hook.py | 2 | ||||
-rw-r--r-- | module/plugins/internal/Hoster.py | 244 | ||||
-rw-r--r-- | module/plugins/internal/MultiAccount.py (renamed from module/plugins/internal/MultiHook.py) | 59 | ||||
-rw-r--r-- | module/plugins/internal/MultiCrypter.py | 4 | ||||
-rw-r--r-- | module/plugins/internal/MultiHoster.py | 14 | ||||
-rw-r--r-- | module/plugins/internal/Plugin.py | 60 | ||||
-rw-r--r-- | module/plugins/internal/SevenZip.py | 6 | ||||
-rw-r--r-- | module/plugins/internal/SimpleCrypter.py | 6 | ||||
-rw-r--r-- | module/plugins/internal/SimpleHoster.py | 51 | ||||
-rw-r--r-- | module/plugins/internal/UnRar.py | 6 | ||||
-rw-r--r-- | module/plugins/internal/UnZip.py | 1 | ||||
-rw-r--r-- | module/plugins/internal/XFSCrypter.py | 18 | ||||
-rw-r--r-- | module/plugins/internal/XFSHoster.py | 26 |
17 files changed, 279 insertions, 229 deletions
diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 5150e88f6..3a252fdfb 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -26,7 +26,6 @@ class Addon(Plugin): __version__ = "0.06" __status__ = "testing" - __config__ = [] #: [("name", "type", "desc", "default")] __threaded__ = [] #@TODO: Remove in 0.4.10 __description__ = """Base addon plugin""" diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 430590421..524b55839 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -17,7 +17,6 @@ class Container(Crypter): __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [] #: [("name", "type", "desc", "default")] __description__ = """Base container decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 2033b67df..ad5bcc74e 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -7,7 +7,7 @@ from module.utils import save_path as safe_filename class Crypter(Hoster): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.08" + __version__ = "0.09" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -83,7 +83,7 @@ class Crypter(Hoster): self.pyload.api.setPackageData(pid, {'password': package_password}) #: Workaround to do not break API addPackage method - set_folder = lambda x: self.pyload.api.setPackageData(pid, {'folder': safe_filename(x) or ""}) + set_folder = lambda x="": self.pyload.api.setPackageData(pid, {'folder': safe_filename(x)}) if use_subfolder: if not subfolder_per_package: @@ -98,4 +98,4 @@ class Crypter(Hoster): self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder}) elif folder_per_package: - set_folder(None) + set_folder() diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index f21fe473c..3ab5d6a0d 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -5,6 +5,7 @@ import re from module.PyFile import PyFile from module.plugins.internal.Plugin import Plugin +from module.utils import fs_encode class ArchiveError(Exception): @@ -22,7 +23,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.34" + __version__ = "0.35" __status__ = "testing" __description__ = """Base extractor plugin""" diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 1f566f824..8ae731a7f 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -9,8 +9,6 @@ class Hook(Addon): __version__ = "0.13" __status__ = "testing" - __config__ = [] #: [("name", "type", "desc", "default")] - __description__ = """Base hook plugin""" __license__ = "GPLv3" __authors__ = [("mkaay" , "mkaay@mkaay.de" ), diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 5d0a64f1a..bc340e78f 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -11,8 +11,8 @@ import urlparse from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, decode, encode, exists, parse_html_form, - parse_html_tag_attr_value, parse_name, + chunks, decode, encode, exists, fixurl, + parse_html_form, parse_html_tag_attr_value, parse_name, replace_patterns, seconds_to_midnight, set_cookie, set_cookies, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename @@ -41,21 +41,27 @@ def create_getInfo(klass): return get_info +#@NOTE: `check_abort` decorator +def check_abort(fn): + + def wrapper(self, *args, **kwargs): + self.check_abort() + return fn(self, *args, **kwargs) + + return wrapper + + class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.28" + __version__ = "0.31" __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [] #: [("name", "type", "desc", "default")] __description__ = """Base hoster plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), - ("spoob" , "spoob@pyload.org" ), - ("mkaay" , "mkaay@mkaay.de" ), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] def __init__(self, pyfile): @@ -74,7 +80,7 @@ class Hoster(Plugin): #: Account handler instance, see :py:class:`Account` self.account = None - self.req = None #: Browser instance, see `network.Browser` + self.user = None #@TODO: Remove in 0.4.10 #: Associated pyfile instance, see `PyFile` self.pyfile = pyfile @@ -98,7 +104,7 @@ class Hoster(Plugin): #: Dict of the amount of retries already made self.retries = {} - self.retry_free = False #@TODO: Recheck in 0.4.10 + self.force_free = False #@TODO: Recheck in 0.4.10 self._setup() self.init() @@ -116,9 +122,10 @@ class Hoster(Plugin): @classmethod def get_info(cls, url="", html=""): + url = fixurl(url, unquote=True) return {'name' : parse_name(url), 'size' : 0, - 'status': 3 if url.strip() else 8, + 'status': 3 if url else 8, 'url' : url} @@ -142,6 +149,11 @@ class Hoster(Plugin): self.last_download = "" self.pyfile.error = "" + try: + self.req.close() + except Exception: + pass + if self.account: self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) self.chunk_limit = -1 #: -1 for unlimited @@ -162,16 +174,16 @@ class Hoster(Plugin): """ self.thread = thread - if self.retry_free: + if self.force_free: self.account = False else: self.load_account() #@TODO: Move to PluginThread in 0.4.10 - self.retry_free = False + self.force_free = False self._setup() + # self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 self.pyfile.setStatus("starting") - self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 self.check_abort() @@ -185,17 +197,17 @@ class Hoster(Plugin): def load_account(self): - if self.req: - self.req.close() - if not self.account: self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) - if self.account: - if not hasattr(self.account, 'user'): #@TODO: Move to `Account` in 0.4.10 - self.account.user = self.account.select()[0] + if not self.account: + self.account = False + self.user = None #@TODO: Remove in 0.4.10 - if not hasattr(self.account, 'logged'): + else: + self.account.choose() + self.user = self.account.user #@TODO: Remove in 0.4.10 + if self.account.user is None: self.account = False @@ -207,12 +219,8 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): - if reconnect: - self.log_info(_("Requesting line reconnection...")) - else: - self.log_debug("Reconnect: %s" % reconnect) - - self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) + self.log_debug("RECONNECT %s required" % ("" if reconnect else "not"), + "Previous wantReconnect: %s" % self.wantReconnect) self.wantReconnect = bool(reconnect) @@ -226,8 +234,8 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_info(_("Waiting %d seconds...") % wait_time) - self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) + self.log_debug("WAIT set to %d seconds" % wait_time, + "Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until @@ -249,13 +257,17 @@ class Hoster(Plugin): self.waiting = True - status = pyfile.status #@NOTE: Remove in 0.4.10 + status = pyfile.status #@NOTE: Recheck in 0.4.10 pyfile.setStatus("waiting") - if not self.wantReconnect or self.account: + self.log_info(_("Waiting %d seconds...") % pyfile.waitUntil - time.time()) + + if self.wantReconnect: + self.log_info(_("Requiring reconnection...")) if self.account: self.log_warning("Ignore reconnection due logged account") + if not self.wantReconnect or self.account: while pyfile.waitUntil > time.time(): self.check_abort() time.sleep(2) @@ -263,24 +275,24 @@ class Hoster(Plugin): else: while pyfile.waitUntil > time.time(): self.check_abort() + self.thread.m.reconnecting.wait(1) if self.thread.m.reconnecting.isSet(): self.waiting = False self.wantReconnect = False raise Reconnect - self.thread.m.reconnecting.wait(2) time.sleep(2) self.waiting = False - pyfile.status = status #@NOTE: Remove in 0.4.10 + pyfile.status = status #@NOTE: Recheck in 0.4.10 def skip(self, msg=""): """ Skip and give msg """ - raise Skip(encode(msg or self.pyfile.error)) #@TODO: Remove `encode` in 0.4.10 + raise Skip(encode(msg or self.pyfile.error or self.pyfile.pluginname)) #@TODO: Remove `encode` in 0.4.10 #@TODO: Remove in 0.4.10 @@ -293,7 +305,7 @@ class Hoster(Plugin): if msg: self.pyfile.error = msg else: - msg = self.pyfile.error + msg = self.pyfile.error or (self.info['error'] if 'error' in self.info else self.pyfile.getStatusName()) raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 @@ -359,30 +371,33 @@ class Hoster(Plugin): if nopremium: if self.premium: - self.retry_free = True + self.force_free = True else: self.fail("%s | %s" % (msg, _("Download was already free"))) raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def fixurl(self, url, baseurl=None): + def fixurl(self, url, baseurl=None, unquote=None): + url = fixurl(url) + if not baseurl: - baseurl = self.pyfile.url + baseurl = fixurl(self.pyfile.url) if not urlparse.urlparse(url).scheme: url_p = urlparse.urlparse(baseurl) baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) url = urlparse.urljoin(baseurl, url) - return url + return fixurl(url, unquote) + @check_abort def load(self, *args, **kwargs): - self.check_abort() return super(Hoster, self).load(*args, **kwargs) + @check_abort def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): """ Downloads the content at url to download folder @@ -396,16 +411,18 @@ class Hoster(Plugin): the filename will be changed if needed :return: The location where the file was saved """ - self.check_abort() - if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, - *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) + + url = self.fixurl(url, unquote=True) self.pyfile.name = parse_name(self.pyfile.name) #: Safe check self.captcha.correct() - self.check_for_same_files() + + if self.pyload.config.get("download", "skip_existing"): + self.check_filedupe() self.pyfile.setStatus("downloading") @@ -441,7 +458,9 @@ class Hoster(Plugin): if finalname != newname: try: - os.rename(fs_join(location, newname), fs_join(location, finalname)) + oldname_enc = fs_join(download_location, newname) + newname_enc = fs_join(download_location, finalname) + os.rename(oldname_enc, newname_enc) except OSError, e: self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e) @@ -463,16 +482,16 @@ class Hoster(Plugin): if not self.pyfile.abort: return - if self.pyfile.hasStatus("failed"): + if self.pyfile.status is 8: self.fail() - elif self.pyfile.hasStatus("skipped"): + elif self.pyfile.status is 4: self.skip(self.pyfile.statusname) - elif self.pyfile.hasStatus("offline"): + elif self.pyfile.status is 1: self.offline() - elif self.pyfile.hasStatus("temp. offline"): + elif self.pyfile.status is 6: self.temp_offline() else: @@ -489,7 +508,8 @@ class Hoster(Plugin): if not self.last_download: return - download_size = os.stat(fs_encode(self.last_download)).st_size + download_location = fs_encode(self.last_download) + download_size = os.stat(download_location).st_size if download_size < 1: self.fail(_("Empty file")) @@ -505,7 +525,7 @@ class Hoster(Plugin): self.log_warning(_("File size is not equal to expected size")) - def check_download(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): + def check_file(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): """ Checks the content of the last downloaded file, re match is saved to `last_check` @@ -552,7 +572,66 @@ class Hoster(Plugin): else: self.log_info(_("File deleted: ") + self.last_download) - self.last_download = "" + self.last_download = "" #: Recheck in 0.4.10 + + + def check_traffic(self): + if not self.account: + return True + + traffic = self.account.get_data(refresh=True)['trafficleft'] + + if traffic is None: + return False + + elif traffic is -1: + return True + + else: + size = self.pyfile.size / 1024 #@TODO: Remove in 0.4.10 + self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic)) #@TODO: Rewrite in 0.4.10 + return size <= traffic + + + def check_filedupe(self): + """ + Checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises Skip: + """ + pack = self.pyfile.package() + + for pyfile in self.pyload.files.cache.values(): + if pyfile is self.pyfile: + continue + + if pyfile.name != self.pyfile.name or pyfile.package().folder != pack.folder: + continue + + if pyfile.status in (0, 5, 7, 12): #: (finished, waiting, starting, downloading) + self.skip(pyfile.pluginname) + + download_folder = self.pyload.config.get("general", "download_folder") + package_folder = pack.folder if self.pyload.config.get("general", "folder_per_package") else "" + download_location = fs_join(download_folder, package_folder, self.pyfile.name) + + if not exists(download_location): + return + + pyfile = self.pyload.db.findDuplicates(self.pyfile.id, package_folder, self.pyfile.name) + if pyfile: + self.skip(pyfile[0]) + + size = os.stat(download_location).st_size + if size >= self.pyfile.size: + self.skip(_("File exists")) + + + #: Deprecated method, use `check_filedupe` instead (Remove in 0.4.10) + def checkForSameFiles(self, *args, **kwargs): + if self.pyload.config.get("download", "skip_existing"): + return self.check_filedupe() def direct_link(self, url, follow_location=None): @@ -598,10 +677,10 @@ class Hoster(Plugin): if 'content-disposition' in header: link = url - elif 'location' in header and header['location']: + elif header.get('location'): location = self.fixurl(header['location'], url) - if 'code' in header and header['code'] == 302: + if header.get('code') == 302: link = location if follow_location: @@ -611,7 +690,7 @@ class Hoster(Plugin): else: extension = os.path.splitext(parse_name(url))[-1] - if 'content-type' in header and header['content-type']: + if header.get('content-type'): mimetype = header['content-type'].split(';')[0].strip() elif extension: @@ -641,63 +720,8 @@ class Hoster(Plugin): return parse_html_form(attr_str, self.html, input_names) - def check_traffic_left(self): - if not self.account: - return True - - traffic = self.account.get_data(self.account.user, True)['trafficleft'] - - if traffic is None: - return False - - elif traffic == -1: - return True - - else: - size = self.pyfile.size / 1024 - self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic)) - return size <= traffic - - def get_password(self): """ Get the password the user provided in the package """ return self.pyfile.package().password or "" - - - #: Deprecated method, use `check_for_same_files` instead (Remove in 0.4.10) - def checkForSameFiles(self, *args, **kwargs): - return self.check_for_same_files(*args, **kwargs) - - - def check_for_same_files(self, starting=False): - """ - Checks if same file was/is downloaded within same package - - :param starting: indicates that the current download is going to start - :raises Skip: - """ - pack = self.pyfile.package() - - for pyfile in self.pyload.files.cache.values(): - if pyfile != self.pyfile and pyfile.name is self.pyfile.name and pyfile.package().folder is pack.folder: - if pyfile.status in (0, 12): #: Finished or downloading - self.skip(pyfile.pluginname) - elif pyfile.status in (5, 7) and starting: #: A download is waiting/starting and was appenrently started before - self.skip(pyfile.pluginname) - - download_folder = self.pyload.config.get("general", "download_folder") - location = fs_join(download_folder, pack.folder, self.pyfile.name) - - if starting and self.pyload.config.get("download", "skip_existing") and exists(location): - size = os.stat(location).st_size - if size >= self.pyfile.size: - self.skip("File exists") - - pyfile = self.pyload.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) - if pyfile: - if exists(location): - self.skip(pyfile[0]) - - self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name) diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiAccount.py index 42a1985b5..b38670ce7 100644 --- a/module/plugins/internal/MultiHook.py +++ b/module/plugins/internal/MultiAccount.py @@ -2,30 +2,29 @@ import re import time -import traceback -from module.plugins.internal.Hook import Hook +from module.plugins.internal.Account import Account from module.utils import decode, remove_chars -class MultiHook(Hook): - __name__ = "MultiHook" - __type__ = "hook" - __version__ = "0.54" +class MultiAccount(Account): + __name__ = "MultiAccount" + __type__ = "account" + __version__ = "0.02" __status__ = "testing" - __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), - ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), - ("reload" , "bool" , "Reload plugin list" , True ), - ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] - __description__ = """Hook plugin for multi hoster/crypter""" + __description__ = """Multi hoster account plugin""" __license__ = "GPLv3" __authors__ = [("pyLoad Team" , "admin@pyload.org" ), ("Walter Purcaro", "vuolter@gmail.com")] - MIN_RELOAD_INTERVAL = 1 * 60 * 60 #: 1 hour + REFRESH_INTERVAL = 1 * 60 * 60 #: 1 hour DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"), (r'bayfiles\.net' , "bayfiles.com" ), @@ -55,6 +54,34 @@ class MultiHook(Hook): (r'^0' , "zero" )] + + + + + + + + + + + + + + + + + + + + + + + + + + + + def init(self): self.plugins = [] self.supported = [] @@ -102,7 +129,7 @@ class MultiHook(Hook): for _i in xrange(5): try: - pluginset = self._plugin_set(self.get_hosters()) + pluginset = self._plugin_set(self.grab_hosters()) break except Exception, e: @@ -110,7 +137,7 @@ class MultiHook(Hook): time.sleep(60) else: self.log_error(_("No hoster list retrieved")) - self.interval = self.MIN_RELOAD_INTERVAL + self.interval = self.REFRESH_INTERVAL return list() try: @@ -144,7 +171,7 @@ class MultiHook(Hook): return set(plugins) - def get_hosters(self): + def grab_hosters(self, user, password, data): """ Load list of supported hoster @@ -160,7 +187,7 @@ class MultiHook(Hook): self.load_account() if self.get_config('reload', True): - self.interval = max(self.get_config('reloadinterval', 12) * 60 * 60, self.MIN_RELOAD_INTERVAL) + self.interval = max(self.get_config('reloadinterval', 12) * 60 * 60, self.REFRESH_INTERVAL) else: self.pyload.scheduler.removeJob(self.cb) self.cb = None diff --git a/module/plugins/internal/MultiCrypter.py b/module/plugins/internal/MultiCrypter.py index ca7b03941..ae8785116 100644 --- a/module/plugins/internal/MultiCrypter.py +++ b/module/plugins/internal/MultiCrypter.py @@ -6,7 +6,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter class MultiCrypter(SimpleCrypter): __name__ = "MultiCrypter" __type__ = "hoster" - __version__ = "0.02" + __version__ = "0.03" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -19,7 +19,7 @@ class MultiCrypter(SimpleCrypter): def init(self): - self.CRYPTER_NAME = self.pyload.pluginManager.crypterPlugins[self.__name__]['name'] + self.PLUGIN_NAME = self.pyload.pluginManager.crypterPlugins[self.__name__]['name'] def _log(self, level, plugintype, pluginname, messages): diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index c0c928a45..fbfab1ade 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.50" + __version__ = "0.51" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -21,21 +21,21 @@ class MultiHoster(SimpleHoster): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - HOSTER_NAME = None + PLUGIN_NAME = None LEECH_HOSTER = False LOGIN_ACCOUNT = True def init(self): - self.HOSTER_NAME = self.pyload.pluginManager.hosterPlugins[self.__name__]['name'] + self.PLUGIN_NAME = self.pyload.pluginManager.hosterPlugins[self.__name__]['name'] def _log(self, level, plugintype, pluginname, messages): return super(MultiHoster, self)._log(level, plugintype, pluginname, - (self.HOSTER_NAME,) + messages) + (self.PLUGIN_NAME,) + messages) def setup(self): @@ -83,11 +83,11 @@ class MultiHoster(SimpleHoster): self.check_errors() self.check_status(getinfo=False) - if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): + if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as premium download...")) self.handle_premium(pyfile) - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as free download...")) self.handle_free(pyfile) @@ -95,7 +95,7 @@ class MultiHoster(SimpleHoster): self.log_info(_("Downloading file...")) self.download(self.link, disposition=self.DISPOSITION) - self.check_file() + self.check_download() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 if self.premium: diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 51192d8c9..f1cd74fcb 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -6,7 +6,6 @@ import datetime import inspect import os import re -import sys import traceback import urllib import urlparse @@ -28,6 +27,11 @@ def decode(string, encoding='utf8'): return unicode(string) +#@TODO: Remove in 0.4.10 +def _decode(*args, **kwargs): + return decode(*args, **kwargs) + + #@TODO: Move to utils in 0.4.10 def encode(string, encoding='utf8'): """ Decode string to utf8 """ @@ -49,18 +53,25 @@ def exists(path): return False -#@TODO: Move to utils in 0.4.10 -def parse_name(url): - url = urllib.unquote(url) - url = url.decode('unicode-escape') - url = html_unescape(url) - url = urllib.quote(url) +#@TODO: Recheck in 0.4.10 +def fixurl(url, unquote=None): + newurl = urllib.unquote(url) + + if unquote is None: + unquote = newurl == url + + newurl = html_unescape(newurl.decode('unicode-escape')).strip() - url_p = urlparse.urlparse(url.strip().rstrip('/')) + return newurl if unquote else urllib.quote(newurl) - name = (url_p.path.split('/')[-1] or - url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or - url_p.netloc.split('.', 1)[0]) + +#@TODO: Recheck in 0.4.10 +def parse_name(string): + path = fixurl(decode(string), unquote=False) + url_p = urlparse.urlparse(path.rstrip('/')) + name = (url_p.path.split('/')[-1] or + url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or + url_p.netloc.split('.', 1)[0]) return urllib.unquote(name) @@ -96,7 +107,7 @@ def seconds_to_midnight(utc=None): else: now = datetime.datetime.utcnow() + datetime.timedelta(hours=utc) - midnight = now.replace(hour=0, minute=10, second=0, microsecond=0) + datetime.timedelta(days=1) + midnight = now.replace(hour=0, minute=1, second=0, microsecond=0) + datetime.timedelta(days=1) return (midnight - now).seconds @@ -173,7 +184,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.37" + __version__ = "0.38" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -199,8 +210,8 @@ class Plugin(object): def _init(self, core): self.pyload = core - self.info = {} #: Provide information in dict here - self.req = None + self.info = {} #: Provide information in dict here + self.req = None #: Browser instance, see `network.Browser` def init(self): @@ -231,8 +242,6 @@ class Plugin(object): def log_warning(self, *args): self._log("warning", self.__type__, self.__name__, args) - if self.pyload.debug: - traceback.print_exc() def log_error(self, *args): @@ -278,7 +287,7 @@ class Plugin(object): return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) - def set_config(self, option, value): + def set_config(self, option, value, plugin=None): """ Set config value for current plugin @@ -286,7 +295,7 @@ class Plugin(object): :param value: :return: """ - self.pyload.config.setPlugin(self.__name__, option, value) + self.pyload.config.setPlugin(plugin or self.__name__, option, value) def get_config(self, option, default="", plugin=None): @@ -347,7 +356,9 @@ class Plugin(object): """ if self.pyload.debug: self.log_debug("LOAD URL " + url, - *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) + + url = fixurl(url, unquote=True) #: Recheck in 0.4.10 if req is None: req = self.req or self.pyload.requestFactory.getRequest(self.__name__) @@ -364,7 +375,7 @@ class Plugin(object): #@TODO: Move to network in 0.4.10 if isinstance(decode, basestring): - res = sys.modules[self.__name__].decode(res, decode) #@TODO: See #1787, use utils.decode() in 0.4.10 + res = _decode(res, decode) #@NOTE: Use `utils.decode()` in 0.4.10 if self.pyload.debug: frame = inspect.currentframe() @@ -409,11 +420,12 @@ class Plugin(object): Clean everything and remove references """ try: + self.req.clearCookies() self.req.close() except Exception: pass - for a in ("pyfile", "thread", "html", "req"): - if hasattr(self, a): - setattr(self, a, None) + for attr in ("account", "html", "pyfile", "req", "thread"): + if hasattr(self, attr): + setattr(self, attr, None) diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index b79256536..9d3a71a21 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -5,12 +5,12 @@ import re import subprocess from module.plugins.internal.UnRar import ArchiveError, CRCError, PasswordError, UnRar, renice -from module.utils import fs_encode, save_join as fs_join +from module.utils import save_join as fs_join class SevenZip(UnRar): __name__ = "SevenZip" - __version__ = "0.15" + __version__ = "0.16" __status__ = "testing" __description__ = """7-Zip extractor plugin""" @@ -128,7 +128,7 @@ class SevenZip(UnRar): args.append("-y") #: Set a password - if "password" in kwargs and kwargs['password']: + if kwargs.get('password'): args.append("-p%s" % kwargs['password']) else: args.append("-p-") diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 8c5d3599d..53d69b2a5 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.62" + __version__ = "0.63" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -81,7 +81,7 @@ class SimpleCrypter(Crypter, SimpleHoster): self.log_debug("Redirect #%d to: %s" % (i, redirect)) header = self.load(redirect, just_header=True) - if 'location' in header and header['location']: + if header.get('location'): self.link = header['location'] else: break @@ -137,7 +137,7 @@ class SimpleCrypter(Crypter, SimpleHoster): try: pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) - except AttributeError: + except (AttributeError, IndexError, ValueError): pages = 1 for p in xrange(2, pages + 1): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 0f030e000..00b63ac12 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -6,7 +6,6 @@ import os import re import time -from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo @@ -14,14 +13,10 @@ from module.plugins.internal.Plugin import Fail, encode, parse_name, replace_pat from module.utils import fixup, fs_encode, parseFileSize as parse_size -#@TODO: Adapt and move to PyFile in 0.4.10 -statusMap = dict((v, k) for k, v in _statusMap.items()) - - class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.86" + __version__ = "1.87" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -124,7 +119,7 @@ class SimpleHoster(Hoster): try: info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: Pattern groups will be saved here - except AttributeError: + except (AttributeError, IndexError): info['pattern'] = {} if not html and not online: @@ -164,7 +159,7 @@ class SimpleHoster(Hoster): if all(True for k in pdict if k not in info['pattern']): info['pattern'].update(pdict) - except AttributeError: + except (AttributeError, IndexError): continue else: @@ -275,11 +270,11 @@ class SimpleHoster(Hoster): if 'status' not in self.info or self.info['status'] is 3: #@TODO: Recheck in 0.4.10 self.check_info() - if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): + if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as premium download...")) self.handle_premium(pyfile) - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as free download...")) self.handle_free(pyfile) @@ -287,7 +282,7 @@ class SimpleHoster(Hoster): self.log_info(_("Downloading file...")) self.download(self.link, disposition=self.DISPOSITION) - self.check_file() + self.check_download() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 if self.get_config('premium_fallback', True) and self.premium: @@ -298,26 +293,26 @@ class SimpleHoster(Hoster): raise Fail(encode(e)) #@TODO: Remove `encode` in 0.4.10 - def check_file(self): - self.log_info(_("Checking file...")) + def check_download(self): + self.log_info(_("Checking downloaded file...")) if self.captcha.task and not self.last_download: self.captcha.invalid() self.retry(10, msg=_("Wrong captcha")) - elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, + elif self.check_file({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, delete=True): self.error(_("Empty file")) else: - if self.get_config('chk_filesize', False) and 'size' in self.info: + if self.get_config('chk_filesize', False) and self.info.get('size'): # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB self.check_filesize(self.info['size'], size_tolerance=10485760) self.log_debug("Using default check rules...") for r, p in self.FILE_ERRORS: - errmsg = self.check_download({r: re.compile(p)}) + errmsg = self.check_file({r: re.compile(p)}) if errmsg is not None: errmsg = errmsg.strip().capitalize() @@ -338,7 +333,6 @@ class SimpleHoster(Hoster): self.check_errors() self.log_info(_("No errors found")) - self.pyfile.error = "" #@TODO: Recheck in 0.4.10 def check_errors(self): @@ -361,7 +355,7 @@ class SimpleHoster(Hoster): try: errmsg = m.group(1).strip() - except AttributeError: + except (AttributeError, IndexError): errmsg = m.group(0).strip() self.info['error'] = re.sub(r'<.*?>', " ", errmsg) @@ -385,7 +379,7 @@ class SimpleHoster(Hoster): try: errmsg = m.group(1).strip() - except AttributeError: + except (AttributeError, IndexError): errmsg = m.group(0).strip() self.info['error'] = re.sub(r'<.*?>', " ", errmsg) @@ -436,7 +430,7 @@ class SimpleHoster(Hoster): try: waitmsg = m.group(1).strip() - except AttributeError: + except (AttributeError, IndexError): waitmsg = m.group(0).strip() wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in @@ -455,22 +449,19 @@ class SimpleHoster(Hoster): self.log_debug("Previous file info: %s" % old_info) try: - status = self.info['status'] or None + status = self.info['status'] or 14 - if status == 1: + if status is 1: self.offline() - elif status == 6: + elif status is 6: self.temp_offline() - elif status == 8: - if 'error' in self.info: - self.fail(self.info['error']) - else: - self.fail(_("File status: " + statusMap[status])) + elif status is 8: + self.fail() finally: - self.log_info(_("File status: ") + (statusMap[status] if status else _("Unknown"))) + self.log_info(_("File status: ") + self.pyfile.getStatusName(status)) def check_name_size(self, getinfo=True): @@ -492,7 +483,7 @@ class SimpleHoster(Hoster): if name and name is not url: self.pyfile.name = name - if 'size' in self.info and self.info['size'] > 0: + if self.info.get('size') > 0: self.pyfile.size = int(self.info['size']) #@TODO: Fix int conversion in 0.4.10 # self.pyfile.sync() diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 88c490750..7efecc5ef 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -8,7 +8,7 @@ from glob import glob from string import digits from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -from module.utils import fs_decode, fs_encode, save_join as fs_join +from module.utils import fs_decode, save_join as fs_join def renice(pid, value): @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.26" + __version__ = "1.27" __status__ = "testing" __description__ = """Rar extractor plugin""" @@ -229,7 +229,7 @@ class UnRar(Extractor): args.append("-y") #: Set a password - if "password" in kwargs and kwargs['password']: + if kwargs.get('password'): args.append("-p%s" % kwargs['password']) else: args.append("-p-") diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index ac197a80d..87cbd568a 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -7,7 +7,6 @@ import sys import zipfile from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -from module.utils import fs_encode class UnZip(Extractor): diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 12a48d4a3..8047fd03e 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -7,7 +7,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.14" + __version__ = "0.15" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -17,7 +17,7 @@ class XFSCrypter(SimpleCrypter): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - HOSTER_DOMAIN = None + PLUGIN_DOMAIN = None URL_REPLACEMENTS = [(r'&?per_page=\d+', ""), (r'[?/&]+$', ""), (r'(.+/[^?]+)$', r'\1?'), (r'$', r'&per_page=10000')] @@ -29,22 +29,22 @@ class XFSCrypter(SimpleCrypter): def prepare(self): - if not self.HOSTER_DOMAIN: + if not self.PLUGIN_DOMAIN: if self.account: account = self.account else: account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") account = self.pyload.accountManager.getAccountPlugin(account_name) - if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: - self.HOSTER_DOMAIN = account.HOSTER_DOMAIN + if account and hasattr(account, "PLUGIN_DOMAIN") and account.PLUGIN_DOMAIN: + self.PLUGIN_DOMAIN = account.PLUGIN_DOMAIN else: - self.fail(_("Missing HOSTER_DOMAIN")) + self.fail(_("Missing PLUGIN_DOMAIN")) if self.COOKIES: - if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: + self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) else: - set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") + set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") return super(XFSCrypter, self).prepare() diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 729c9a0ee..8af6b83ed 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.60" + __version__ = "0.61" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -26,7 +26,7 @@ class XFSHoster(SimpleHoster): ("Walter Purcaro", "vuolter@gmail.com" )] - HOSTER_DOMAIN = None + PLUGIN_DOMAIN = None LEECH_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... @@ -61,26 +61,26 @@ class XFSHoster(SimpleHoster): """ Initialize important variables """ - if not self.HOSTER_DOMAIN: + if not self.PLUGIN_DOMAIN: if self.account: account = self.account else: account = self.pyload.accountManager.getAccountPlugin(self.__name__) - if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: - self.HOSTER_DOMAIN = account.HOSTER_DOMAIN + if account and hasattr(account, "PLUGIN_DOMAIN") and account.PLUGIN_DOMAIN: + self.PLUGIN_DOMAIN = account.PLUGIN_DOMAIN else: - self.fail(_("Missing HOSTER_DOMAIN")) + self.fail(_("Missing PLUGIN_DOMAIN")) if self.COOKIES: - if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: + self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) else: - set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") + set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") if not self.LINK_PATTERN: pattern = r'(?:file: "(.+?)"|(https?://(?:www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<])' - self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.') + self.LINK_PATTERN = pattern % self.PLUGIN_DOMAIN.replace('.', '\.') super(XFSHoster, self).prepare() @@ -129,7 +129,7 @@ class XFSHoster(SimpleHoster): self.fail(_("Only registered or premium users can use url leech feature")) #: Only tested with easybytez.com - self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) + self.html = self.load("http://www.%s/" % self.PLUGIN_DOMAIN) action, inputs = self.parse_html_form() @@ -244,7 +244,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) - except AttributeError: + except (AttributeError, IndexError): captcha_key = recaptcha.detect_key() else: @@ -258,7 +258,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) - except AttributeError: + except (AttributeError, IndexError): captcha_key = solvemedia.detect_key() else: |