From 8af39bfa2c409d4fd07d430485117bf60fdcc935 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 29 Sep 2015 21:43:22 +0200 Subject: Update internal plugins --- module/plugins/internal/Addon.py | 1 - module/plugins/internal/Container.py | 1 - module/plugins/internal/Crypter.py | 6 +- module/plugins/internal/Extractor.py | 3 +- module/plugins/internal/Hook.py | 2 - module/plugins/internal/Hoster.py | 244 ++++++++++++++------------ module/plugins/internal/MultiAccount.py | 287 +++++++++++++++++++++++++++++++ module/plugins/internal/MultiCrypter.py | 4 +- module/plugins/internal/MultiHook.py | 260 ---------------------------- module/plugins/internal/MultiHoster.py | 14 +- module/plugins/internal/Plugin.py | 60 ++++--- module/plugins/internal/SevenZip.py | 6 +- module/plugins/internal/SimpleCrypter.py | 6 +- module/plugins/internal/SimpleHoster.py | 51 +++--- module/plugins/internal/UnRar.py | 6 +- module/plugins/internal/UnZip.py | 1 - module/plugins/internal/XFSCrypter.py | 18 +- module/plugins/internal/XFSHoster.py | 26 +-- 18 files changed, 523 insertions(+), 473 deletions(-) create mode 100644 module/plugins/internal/MultiAccount.py delete mode 100644 module/plugins/internal/MultiHook.py (limited to 'module/plugins') diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 5150e88f6..3a252fdfb 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -26,7 +26,6 @@ class Addon(Plugin): __version__ = "0.06" __status__ = "testing" - __config__ = [] #: [("name", "type", "desc", "default")] __threaded__ = [] #@TODO: Remove in 0.4.10 __description__ = """Base addon plugin""" diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index 430590421..524b55839 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -17,7 +17,6 @@ class Container(Crypter): __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [] #: [("name", "type", "desc", "default")] __description__ = """Base container decrypter plugin""" __license__ = "GPLv3" diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py index 2033b67df..ad5bcc74e 100644 --- a/module/plugins/internal/Crypter.py +++ b/module/plugins/internal/Crypter.py @@ -7,7 +7,7 @@ from module.utils import save_path as safe_filename class Crypter(Hoster): __name__ = "Crypter" __type__ = "crypter" - __version__ = "0.08" + __version__ = "0.09" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -83,7 +83,7 @@ class Crypter(Hoster): self.pyload.api.setPackageData(pid, {'password': package_password}) #: Workaround to do not break API addPackage method - set_folder = lambda x: self.pyload.api.setPackageData(pid, {'folder': safe_filename(x) or ""}) + set_folder = lambda x="": self.pyload.api.setPackageData(pid, {'folder': safe_filename(x)}) if use_subfolder: if not subfolder_per_package: @@ -98,4 +98,4 @@ class Crypter(Hoster): self.log_debug("Set package %(name)s folder to: %(folder)s" % {'name': name, 'folder': folder}) elif folder_per_package: - set_folder(None) + set_folder() diff --git a/module/plugins/internal/Extractor.py b/module/plugins/internal/Extractor.py index f21fe473c..3ab5d6a0d 100644 --- a/module/plugins/internal/Extractor.py +++ b/module/plugins/internal/Extractor.py @@ -5,6 +5,7 @@ import re from module.PyFile import PyFile from module.plugins.internal.Plugin import Plugin +from module.utils import fs_encode class ArchiveError(Exception): @@ -22,7 +23,7 @@ class PasswordError(Exception): class Extractor(Plugin): __name__ = "Extractor" __type__ = "extractor" - __version__ = "0.34" + __version__ = "0.35" __status__ = "testing" __description__ = """Base extractor plugin""" diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py index 1f566f824..8ae731a7f 100644 --- a/module/plugins/internal/Hook.py +++ b/module/plugins/internal/Hook.py @@ -9,8 +9,6 @@ class Hook(Addon): __version__ = "0.13" __status__ = "testing" - __config__ = [] #: [("name", "type", "desc", "default")] - __description__ = """Base hook plugin""" __license__ = "GPLv3" __authors__ = [("mkaay" , "mkaay@mkaay.de" ), diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 5d0a64f1a..bc340e78f 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -11,8 +11,8 @@ import urlparse from module.plugins.internal.Captcha import Captcha from module.plugins.internal.Plugin import (Plugin, Abort, Fail, Reconnect, Retry, Skip, - chunks, decode, encode, exists, parse_html_form, - parse_html_tag_attr_value, parse_name, + chunks, decode, encode, exists, fixurl, + parse_html_form, parse_html_tag_attr_value, parse_name, replace_patterns, seconds_to_midnight, set_cookie, set_cookies, timestamp) from module.utils import fs_decode, fs_encode, save_join as fs_join, save_path as safe_filename @@ -41,21 +41,27 @@ def create_getInfo(klass): return get_info +#@NOTE: `check_abort` decorator +def check_abort(fn): + + def wrapper(self, *args, **kwargs): + self.check_abort() + return fn(self, *args, **kwargs) + + return wrapper + + class Hoster(Plugin): __name__ = "Hoster" __type__ = "hoster" - __version__ = "0.28" + __version__ = "0.31" __status__ = "testing" __pattern__ = r'^unmatchable$' - __config__ = [] #: [("name", "type", "desc", "default")] __description__ = """Base hoster plugin""" __license__ = "GPLv3" - __authors__ = [("RaNaN" , "RaNaN@pyload.org" ), - ("spoob" , "spoob@pyload.org" ), - ("mkaay" , "mkaay@mkaay.de" ), - ("Walter Purcaro", "vuolter@gmail.com")] + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] def __init__(self, pyfile): @@ -74,7 +80,7 @@ class Hoster(Plugin): #: Account handler instance, see :py:class:`Account` self.account = None - self.req = None #: Browser instance, see `network.Browser` + self.user = None #@TODO: Remove in 0.4.10 #: Associated pyfile instance, see `PyFile` self.pyfile = pyfile @@ -98,7 +104,7 @@ class Hoster(Plugin): #: Dict of the amount of retries already made self.retries = {} - self.retry_free = False #@TODO: Recheck in 0.4.10 + self.force_free = False #@TODO: Recheck in 0.4.10 self._setup() self.init() @@ -116,9 +122,10 @@ class Hoster(Plugin): @classmethod def get_info(cls, url="", html=""): + url = fixurl(url, unquote=True) return {'name' : parse_name(url), 'size' : 0, - 'status': 3 if url.strip() else 8, + 'status': 3 if url else 8, 'url' : url} @@ -142,6 +149,11 @@ class Hoster(Plugin): self.last_download = "" self.pyfile.error = "" + try: + self.req.close() + except Exception: + pass + if self.account: self.req = self.pyload.requestFactory.getRequest(self.__name__, self.account.user) self.chunk_limit = -1 #: -1 for unlimited @@ -162,16 +174,16 @@ class Hoster(Plugin): """ self.thread = thread - if self.retry_free: + if self.force_free: self.account = False else: self.load_account() #@TODO: Move to PluginThread in 0.4.10 - self.retry_free = False + self.force_free = False self._setup() + # self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 self.pyfile.setStatus("starting") - self.pyload.hookManager.downloadPreparing(self.pyfile) #@TODO: Recheck in 0.4.10 self.check_abort() @@ -185,17 +197,17 @@ class Hoster(Plugin): def load_account(self): - if self.req: - self.req.close() - if not self.account: self.account = self.pyload.accountManager.getAccountPlugin(self.__name__) - if self.account: - if not hasattr(self.account, 'user'): #@TODO: Move to `Account` in 0.4.10 - self.account.user = self.account.select()[0] + if not self.account: + self.account = False + self.user = None #@TODO: Remove in 0.4.10 - if not hasattr(self.account, 'logged'): + else: + self.account.choose() + self.user = self.account.user #@TODO: Remove in 0.4.10 + if self.account.user is None: self.account = False @@ -207,12 +219,8 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): - if reconnect: - self.log_info(_("Requesting line reconnection...")) - else: - self.log_debug("Reconnect: %s" % reconnect) - - self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) + self.log_debug("RECONNECT %s required" % ("" if reconnect else "not"), + "Previous wantReconnect: %s" % self.wantReconnect) self.wantReconnect = bool(reconnect) @@ -226,8 +234,8 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_info(_("Waiting %d seconds...") % wait_time) - self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) + self.log_debug("WAIT set to %d seconds" % wait_time, + "Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until @@ -249,13 +257,17 @@ class Hoster(Plugin): self.waiting = True - status = pyfile.status #@NOTE: Remove in 0.4.10 + status = pyfile.status #@NOTE: Recheck in 0.4.10 pyfile.setStatus("waiting") - if not self.wantReconnect or self.account: + self.log_info(_("Waiting %d seconds...") % pyfile.waitUntil - time.time()) + + if self.wantReconnect: + self.log_info(_("Requiring reconnection...")) if self.account: self.log_warning("Ignore reconnection due logged account") + if not self.wantReconnect or self.account: while pyfile.waitUntil > time.time(): self.check_abort() time.sleep(2) @@ -263,24 +275,24 @@ class Hoster(Plugin): else: while pyfile.waitUntil > time.time(): self.check_abort() + self.thread.m.reconnecting.wait(1) if self.thread.m.reconnecting.isSet(): self.waiting = False self.wantReconnect = False raise Reconnect - self.thread.m.reconnecting.wait(2) time.sleep(2) self.waiting = False - pyfile.status = status #@NOTE: Remove in 0.4.10 + pyfile.status = status #@NOTE: Recheck in 0.4.10 def skip(self, msg=""): """ Skip and give msg """ - raise Skip(encode(msg or self.pyfile.error)) #@TODO: Remove `encode` in 0.4.10 + raise Skip(encode(msg or self.pyfile.error or self.pyfile.pluginname)) #@TODO: Remove `encode` in 0.4.10 #@TODO: Remove in 0.4.10 @@ -293,7 +305,7 @@ class Hoster(Plugin): if msg: self.pyfile.error = msg else: - msg = self.pyfile.error + msg = self.pyfile.error or (self.info['error'] if 'error' in self.info else self.pyfile.getStatusName()) raise Fail(encode(msg)) #@TODO: Remove `encode` in 0.4.10 @@ -359,30 +371,33 @@ class Hoster(Plugin): if nopremium: if self.premium: - self.retry_free = True + self.force_free = True else: self.fail("%s | %s" % (msg, _("Download was already free"))) raise Retry(encode(msg)) #@TODO: Remove `encode` in 0.4.10 - def fixurl(self, url, baseurl=None): + def fixurl(self, url, baseurl=None, unquote=None): + url = fixurl(url) + if not baseurl: - baseurl = self.pyfile.url + baseurl = fixurl(self.pyfile.url) if not urlparse.urlparse(url).scheme: url_p = urlparse.urlparse(baseurl) baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) url = urlparse.urljoin(baseurl, url) - return url + return fixurl(url, unquote) + @check_abort def load(self, *args, **kwargs): - self.check_abort() return super(Hoster, self).load(*args, **kwargs) + @check_abort def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=True): """ Downloads the content at url to download folder @@ -396,16 +411,18 @@ class Hoster(Plugin): the filename will be changed if needed :return: The location where the file was saved """ - self.check_abort() - if self.pyload.debug: self.log_debug("DOWNLOAD URL " + url, - *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) + + url = self.fixurl(url, unquote=True) self.pyfile.name = parse_name(self.pyfile.name) #: Safe check self.captcha.correct() - self.check_for_same_files() + + if self.pyload.config.get("download", "skip_existing"): + self.check_filedupe() self.pyfile.setStatus("downloading") @@ -441,7 +458,9 @@ class Hoster(Plugin): if finalname != newname: try: - os.rename(fs_join(location, newname), fs_join(location, finalname)) + oldname_enc = fs_join(download_location, newname) + newname_enc = fs_join(download_location, finalname) + os.rename(oldname_enc, newname_enc) except OSError, e: self.log_warning(_("Error renaming `%s` to `%s`") % (newname, finalname), e) @@ -463,16 +482,16 @@ class Hoster(Plugin): if not self.pyfile.abort: return - if self.pyfile.hasStatus("failed"): + if self.pyfile.status is 8: self.fail() - elif self.pyfile.hasStatus("skipped"): + elif self.pyfile.status is 4: self.skip(self.pyfile.statusname) - elif self.pyfile.hasStatus("offline"): + elif self.pyfile.status is 1: self.offline() - elif self.pyfile.hasStatus("temp. offline"): + elif self.pyfile.status is 6: self.temp_offline() else: @@ -489,7 +508,8 @@ class Hoster(Plugin): if not self.last_download: return - download_size = os.stat(fs_encode(self.last_download)).st_size + download_location = fs_encode(self.last_download) + download_size = os.stat(download_location).st_size if download_size < 1: self.fail(_("Empty file")) @@ -505,7 +525,7 @@ class Hoster(Plugin): self.log_warning(_("File size is not equal to expected size")) - def check_download(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): + def check_file(self, rules, delete=False, read_size=1048576, file_size=0, size_tolerance=1024): """ Checks the content of the last downloaded file, re match is saved to `last_check` @@ -552,7 +572,66 @@ class Hoster(Plugin): else: self.log_info(_("File deleted: ") + self.last_download) - self.last_download = "" + self.last_download = "" #: Recheck in 0.4.10 + + + def check_traffic(self): + if not self.account: + return True + + traffic = self.account.get_data(refresh=True)['trafficleft'] + + if traffic is None: + return False + + elif traffic is -1: + return True + + else: + size = self.pyfile.size / 1024 #@TODO: Remove in 0.4.10 + self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic)) #@TODO: Rewrite in 0.4.10 + return size <= traffic + + + def check_filedupe(self): + """ + Checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises Skip: + """ + pack = self.pyfile.package() + + for pyfile in self.pyload.files.cache.values(): + if pyfile is self.pyfile: + continue + + if pyfile.name != self.pyfile.name or pyfile.package().folder != pack.folder: + continue + + if pyfile.status in (0, 5, 7, 12): #: (finished, waiting, starting, downloading) + self.skip(pyfile.pluginname) + + download_folder = self.pyload.config.get("general", "download_folder") + package_folder = pack.folder if self.pyload.config.get("general", "folder_per_package") else "" + download_location = fs_join(download_folder, package_folder, self.pyfile.name) + + if not exists(download_location): + return + + pyfile = self.pyload.db.findDuplicates(self.pyfile.id, package_folder, self.pyfile.name) + if pyfile: + self.skip(pyfile[0]) + + size = os.stat(download_location).st_size + if size >= self.pyfile.size: + self.skip(_("File exists")) + + + #: Deprecated method, use `check_filedupe` instead (Remove in 0.4.10) + def checkForSameFiles(self, *args, **kwargs): + if self.pyload.config.get("download", "skip_existing"): + return self.check_filedupe() def direct_link(self, url, follow_location=None): @@ -598,10 +677,10 @@ class Hoster(Plugin): if 'content-disposition' in header: link = url - elif 'location' in header and header['location']: + elif header.get('location'): location = self.fixurl(header['location'], url) - if 'code' in header and header['code'] == 302: + if header.get('code') == 302: link = location if follow_location: @@ -611,7 +690,7 @@ class Hoster(Plugin): else: extension = os.path.splitext(parse_name(url))[-1] - if 'content-type' in header and header['content-type']: + if header.get('content-type'): mimetype = header['content-type'].split(';')[0].strip() elif extension: @@ -641,63 +720,8 @@ class Hoster(Plugin): return parse_html_form(attr_str, self.html, input_names) - def check_traffic_left(self): - if not self.account: - return True - - traffic = self.account.get_data(self.account.user, True)['trafficleft'] - - if traffic is None: - return False - - elif traffic == -1: - return True - - else: - size = self.pyfile.size / 1024 - self.log_info(_("Filesize: %s KiB, Traffic left for user %s: %s KiB") % (size, self.account.user, traffic)) - return size <= traffic - - def get_password(self): """ Get the password the user provided in the package """ return self.pyfile.package().password or "" - - - #: Deprecated method, use `check_for_same_files` instead (Remove in 0.4.10) - def checkForSameFiles(self, *args, **kwargs): - return self.check_for_same_files(*args, **kwargs) - - - def check_for_same_files(self, starting=False): - """ - Checks if same file was/is downloaded within same package - - :param starting: indicates that the current download is going to start - :raises Skip: - """ - pack = self.pyfile.package() - - for pyfile in self.pyload.files.cache.values(): - if pyfile != self.pyfile and pyfile.name is self.pyfile.name and pyfile.package().folder is pack.folder: - if pyfile.status in (0, 12): #: Finished or downloading - self.skip(pyfile.pluginname) - elif pyfile.status in (5, 7) and starting: #: A download is waiting/starting and was appenrently started before - self.skip(pyfile.pluginname) - - download_folder = self.pyload.config.get("general", "download_folder") - location = fs_join(download_folder, pack.folder, self.pyfile.name) - - if starting and self.pyload.config.get("download", "skip_existing") and exists(location): - size = os.stat(location).st_size - if size >= self.pyfile.size: - self.skip("File exists") - - pyfile = self.pyload.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) - if pyfile: - if exists(location): - self.skip(pyfile[0]) - - self.log_debug("File %s not skipped, because it does not exists." % self.pyfile.name) diff --git a/module/plugins/internal/MultiAccount.py b/module/plugins/internal/MultiAccount.py new file mode 100644 index 000000000..b38670ce7 --- /dev/null +++ b/module/plugins/internal/MultiAccount.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- + +import re +import time + +from module.plugins.internal.Account import Account +from module.utils import decode, remove_chars + + +class MultiAccount(Account): + __name__ = "MultiAccount" + __type__ = "account" + __version__ = "0.02" + __status__ = "testing" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Multi hoster account plugin""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team" , "admin@pyload.org" ), + ("Walter Purcaro", "vuolter@gmail.com")] + + + REFRESH_INTERVAL = 1 * 60 * 60 #: 1 hour + + DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"), + (r'bayfiles\.net' , "bayfiles.com" ), + (r'cloudnator\.com' , "shragle.com" ), + (r'dfiles\.eu' , "depositfiles.com" ), + (r'easy-share\.com' , "crocko.com" ), + (r'freakshare\.net' , "freakshare.com" ), + (r'hellshare\.com' , "hellshare.cz" ), + (r'ifile\.it' , "filecloud.io" ), + (r'nowdownload\.\w+', "nowdownload.sx" ), + (r'nowvideo\.\w+' , "nowvideo.sx" ), + (r'putlocker\.com' , "firedrive.com" ), + (r'share-?rapid\.cz', "multishare.cz" ), + (r'ul\.to' , "uploaded.to" ), + (r'uploaded\.net' , "uploaded.to" ), + (r'uploadhero\.co' , "uploadhero.com" ), + (r'zshares\.net' , "zshare.net" ), + (r'^1' , "one" ), + (r'^2' , "two" ), + (r'^3' , "three" ), + (r'^4' , "four" ), + (r'^5' , "five" ), + (r'^6' , "six" ), + (r'^7' , "seven" ), + (r'^8' , "eight" ), + (r'^9' , "nine" ), + (r'^0' , "zero" )] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + def init(self): + self.plugins = [] + self.supported = [] + self.new_supported = [] + + self.account = None + self.pluginclass = None + self.pluginmodule = None + self.pluginname = None + self.plugintype = None + + self.init_plugin() + + + def init_plugin(self): + self.pluginname = self.__name__.rsplit("Hook", 1)[0] + plugin, self.plugintype = self.pyload.pluginManager.findPlugin(self.pluginname) + + if plugin: + self.pluginmodule = self.pyload.pluginManager.loadModule(self.plugintype, self.pluginname) + self.pluginclass = getattr(self.pluginmodule, self.pluginname) + else: + self.log_warning(_("Hook plugin will be deactivated due missing plugin reference")) + self.set_config('activated', False) + + + def load_account(self): + self.account = self.pyload.accountManager.getAccountPlugin(self.pluginname) + + if self.account and not self.account.select()[0]: + self.account = False + + if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: + self.log_warning(_("Hook plugin will be deactivated due missing account reference")) + self.set_config('activated', False) + + + def activate(self): + self.init_periodical(threaded=True) + + + def plugins_cached(self): + if self.plugins: + return self.plugins + + for _i in xrange(5): + try: + pluginset = self._plugin_set(self.grab_hosters()) + break + + except Exception, e: + self.log_warning(e, _("Waiting 1 minute and retry")) + time.sleep(60) + else: + self.log_error(_("No hoster list retrieved")) + self.interval = self.REFRESH_INTERVAL + return list() + + try: + configmode = self.get_config('pluginmode', 'all') + if configmode in ("listed", "unlisted"): + pluginlist = self.get_config('pluginlist', '').replace('|', ',').replace(';', ',').split(',') + configset = self._plugin_set(pluginlist) + + if configmode == "listed": + pluginset &= configset + else: + pluginset -= configset + + except Exception, e: + self.log_error(e) + + self.plugins = list(pluginset) + + return self.plugins + + + def _plugin_set(self, plugins): + regexp = re.compile(r'^[\w\-.^_]{3,63}\.[a-zA-Z]{2,}$', re.U) + plugins = [decode(p.strip()).lower() for p in plugins if regexp.match(p.strip())] + + for r in self.DOMAIN_REPLACEMENTS: + rf, rt = r + repr = re.compile(rf, re.I|re.U) + plugins = [re.sub(rf, rt, p) if repr.match(p) else p for p in plugins] + + return set(plugins) + + + def grab_hosters(self, user, password, data): + """ + Load list of supported hoster + + :return: List of domain names + """ + raise NotImplementedError + + + def periodical(self): + """ + Reload plugin list periodically + """ + self.load_account() + + if self.get_config('reload', True): + self.interval = max(self.get_config('reloadinterval', 12) * 60 * 60, self.REFRESH_INTERVAL) + else: + self.pyload.scheduler.removeJob(self.cb) + self.cb = None + + self.log_info(_("Reloading supported %s list") % self.plugintype) + + old_supported = self.supported + + self.supported = [] + self.new_supported = [] + self.plugins = [] + + self.override_plugins() + + old_supported = [plugin for plugin in old_supported if plugin not in self.supported] + + if old_supported: + self.log_debug("Unload: %s" % ", ".join(old_supported)) + for plugin in old_supported: + self.unload_plugin(plugin) + + + def override_plugins(self): + excludedList = [] + + if self.plugintype == "hoster": + pluginMap = dict((name.lower(), name) for name in self.pyload.pluginManager.hosterPlugins.keys()) + accountList = [account.type.lower() for account in self.pyload.api.getAccounts(False) if account.valid and account.premium] + else: + pluginMap = {} + accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.pyload.pluginManager.crypterPlugins.keys()] + + for plugin in self.plugins_cached(): + name = remove_chars(plugin, "-.") + + if name in accountList: + excludedList.append(plugin) + else: + if name in pluginMap: + self.supported.append(pluginMap[name]) + else: + self.new_supported.append(plugin) + + if not self.supported and not self.new_supported: + self.log_error(_("No %s loaded") % self.plugintype) + return + + #: Inject plugin plugin + self.log_debug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) + + for plugin in self.supported: + hdict = self.pyload.pluginManager.plugins[self.plugintype][plugin] + hdict['new_module'] = self.pluginmodule + hdict['new_name'] = self.pluginname + + if excludedList: + self.log_info(_("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excludedList)))) + + if self.new_supported: + plugins = sorted(self.new_supported) + + self.log_debug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) + + #: Create new regexp + regexp = r'.*(?P%s).*' % "|".join(x.replace('.', '\.') for x in plugins) + if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and "://" in self.pluginclass.__pattern__: + regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) + + self.log_debug("Regexp: %s" % regexp) + + hdict = self.pyload.pluginManager.plugins[self.plugintype][self.pluginname] + hdict['pattern'] = regexp + hdict['re'] = re.compile(regexp) + + + def unload_plugin(self, plugin): + hdict = self.pyload.pluginManager.plugins[self.plugintype][plugin] + if "module" in hdict: + hdict.pop('module', None) + + if "new_module" in hdict: + hdict.pop('new_module', None) + hdict.pop('new_name', None) + + + def deactivate(self): + """ + Remove override for all plugins. Scheduler job is removed by hookmanager + """ + for plugin in self.supported: + self.unload_plugin(plugin) + + #: Reset pattern + hdict = self.pyload.pluginManager.plugins[self.plugintype][self.pluginname] + + hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') + hdict['re'] = re.compile(hdict['pattern']) diff --git a/module/plugins/internal/MultiCrypter.py b/module/plugins/internal/MultiCrypter.py index ca7b03941..ae8785116 100644 --- a/module/plugins/internal/MultiCrypter.py +++ b/module/plugins/internal/MultiCrypter.py @@ -6,7 +6,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter class MultiCrypter(SimpleCrypter): __name__ = "MultiCrypter" __type__ = "hoster" - __version__ = "0.02" + __version__ = "0.03" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -19,7 +19,7 @@ class MultiCrypter(SimpleCrypter): def init(self): - self.CRYPTER_NAME = self.pyload.pluginManager.crypterPlugins[self.__name__]['name'] + self.PLUGIN_NAME = self.pyload.pluginManager.crypterPlugins[self.__name__]['name'] def _log(self, level, plugintype, pluginname, messages): diff --git a/module/plugins/internal/MultiHook.py b/module/plugins/internal/MultiHook.py deleted file mode 100644 index 42a1985b5..000000000 --- a/module/plugins/internal/MultiHook.py +++ /dev/null @@ -1,260 +0,0 @@ -# -*- coding: utf-8 -*- - -import re -import time -import traceback - -from module.plugins.internal.Hook import Hook -from module.utils import decode, remove_chars - - -class MultiHook(Hook): - __name__ = "MultiHook" - __type__ = "hook" - __version__ = "0.54" - __status__ = "testing" - - __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), - ("pluginlist" , "str" , "Plugin list (comma separated)", "" ), - ("reload" , "bool" , "Reload plugin list" , True ), - ("reloadinterval", "int" , "Reload interval in hours" , 12 )] - - __description__ = """Hook plugin for multi hoster/crypter""" - __license__ = "GPLv3" - __authors__ = [("pyLoad Team" , "admin@pyload.org" ), - ("Walter Purcaro", "vuolter@gmail.com")] - - - MIN_RELOAD_INTERVAL = 1 * 60 * 60 #: 1 hour - - DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"), - (r'bayfiles\.net' , "bayfiles.com" ), - (r'cloudnator\.com' , "shragle.com" ), - (r'dfiles\.eu' , "depositfiles.com" ), - (r'easy-share\.com' , "crocko.com" ), - (r'freakshare\.net' , "freakshare.com" ), - (r'hellshare\.com' , "hellshare.cz" ), - (r'ifile\.it' , "filecloud.io" ), - (r'nowdownload\.\w+', "nowdownload.sx" ), - (r'nowvideo\.\w+' , "nowvideo.sx" ), - (r'putlocker\.com' , "firedrive.com" ), - (r'share-?rapid\.cz', "multishare.cz" ), - (r'ul\.to' , "uploaded.to" ), - (r'uploaded\.net' , "uploaded.to" ), - (r'uploadhero\.co' , "uploadhero.com" ), - (r'zshares\.net' , "zshare.net" ), - (r'^1' , "one" ), - (r'^2' , "two" ), - (r'^3' , "three" ), - (r'^4' , "four" ), - (r'^5' , "five" ), - (r'^6' , "six" ), - (r'^7' , "seven" ), - (r'^8' , "eight" ), - (r'^9' , "nine" ), - (r'^0' , "zero" )] - - - def init(self): - self.plugins = [] - self.supported = [] - self.new_supported = [] - - self.account = None - self.pluginclass = None - self.pluginmodule = None - self.pluginname = None - self.plugintype = None - - self.init_plugin() - - - def init_plugin(self): - self.pluginname = self.__name__.rsplit("Hook", 1)[0] - plugin, self.plugintype = self.pyload.pluginManager.findPlugin(self.pluginname) - - if plugin: - self.pluginmodule = self.pyload.pluginManager.loadModule(self.plugintype, self.pluginname) - self.pluginclass = getattr(self.pluginmodule, self.pluginname) - else: - self.log_warning(_("Hook plugin will be deactivated due missing plugin reference")) - self.set_config('activated', False) - - - def load_account(self): - self.account = self.pyload.accountManager.getAccountPlugin(self.pluginname) - - if self.account and not self.account.select()[0]: - self.account = False - - if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: - self.log_warning(_("Hook plugin will be deactivated due missing account reference")) - self.set_config('activated', False) - - - def activate(self): - self.init_periodical(threaded=True) - - - def plugins_cached(self): - if self.plugins: - return self.plugins - - for _i in xrange(5): - try: - pluginset = self._plugin_set(self.get_hosters()) - break - - except Exception, e: - self.log_warning(e, _("Waiting 1 minute and retry")) - time.sleep(60) - else: - self.log_error(_("No hoster list retrieved")) - self.interval = self.MIN_RELOAD_INTERVAL - return list() - - try: - configmode = self.get_config('pluginmode', 'all') - if configmode in ("listed", "unlisted"): - pluginlist = self.get_config('pluginlist', '').replace('|', ',').replace(';', ',').split(',') - configset = self._plugin_set(pluginlist) - - if configmode == "listed": - pluginset &= configset - else: - pluginset -= configset - - except Exception, e: - self.log_error(e) - - self.plugins = list(pluginset) - - return self.plugins - - - def _plugin_set(self, plugins): - regexp = re.compile(r'^[\w\-.^_]{3,63}\.[a-zA-Z]{2,}$', re.U) - plugins = [decode(p.strip()).lower() for p in plugins if regexp.match(p.strip())] - - for r in self.DOMAIN_REPLACEMENTS: - rf, rt = r - repr = re.compile(rf, re.I|re.U) - plugins = [re.sub(rf, rt, p) if repr.match(p) else p for p in plugins] - - return set(plugins) - - - def get_hosters(self): - """ - Load list of supported hoster - - :return: List of domain names - """ - raise NotImplementedError - - - def periodical(self): - """ - Reload plugin list periodically - """ - self.load_account() - - if self.get_config('reload', True): - self.interval = max(self.get_config('reloadinterval', 12) * 60 * 60, self.MIN_RELOAD_INTERVAL) - else: - self.pyload.scheduler.removeJob(self.cb) - self.cb = None - - self.log_info(_("Reloading supported %s list") % self.plugintype) - - old_supported = self.supported - - self.supported = [] - self.new_supported = [] - self.plugins = [] - - self.override_plugins() - - old_supported = [plugin for plugin in old_supported if plugin not in self.supported] - - if old_supported: - self.log_debug("Unload: %s" % ", ".join(old_supported)) - for plugin in old_supported: - self.unload_plugin(plugin) - - - def override_plugins(self): - excludedList = [] - - if self.plugintype == "hoster": - pluginMap = dict((name.lower(), name) for name in self.pyload.pluginManager.hosterPlugins.keys()) - accountList = [account.type.lower() for account in self.pyload.api.getAccounts(False) if account.valid and account.premium] - else: - pluginMap = {} - accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.pyload.pluginManager.crypterPlugins.keys()] - - for plugin in self.plugins_cached(): - name = remove_chars(plugin, "-.") - - if name in accountList: - excludedList.append(plugin) - else: - if name in pluginMap: - self.supported.append(pluginMap[name]) - else: - self.new_supported.append(plugin) - - if not self.supported and not self.new_supported: - self.log_error(_("No %s loaded") % self.plugintype) - return - - #: Inject plugin plugin - self.log_debug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) - - for plugin in self.supported: - hdict = self.pyload.pluginManager.plugins[self.plugintype][plugin] - hdict['new_module'] = self.pluginmodule - hdict['new_name'] = self.pluginname - - if excludedList: - self.log_info(_("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excludedList)))) - - if self.new_supported: - plugins = sorted(self.new_supported) - - self.log_debug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) - - #: Create new regexp - regexp = r'.*(?P%s).*' % "|".join(x.replace('.', '\.') for x in plugins) - if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and "://" in self.pluginclass.__pattern__: - regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) - - self.log_debug("Regexp: %s" % regexp) - - hdict = self.pyload.pluginManager.plugins[self.plugintype][self.pluginname] - hdict['pattern'] = regexp - hdict['re'] = re.compile(regexp) - - - def unload_plugin(self, plugin): - hdict = self.pyload.pluginManager.plugins[self.plugintype][plugin] - if "module" in hdict: - hdict.pop('module', None) - - if "new_module" in hdict: - hdict.pop('new_module', None) - hdict.pop('new_name', None) - - - def deactivate(self): - """ - Remove override for all plugins. Scheduler job is removed by hookmanager - """ - for plugin in self.supported: - self.unload_plugin(plugin) - - #: Reset pattern - hdict = self.pyload.pluginManager.plugins[self.plugintype][self.pluginname] - - hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') - hdict['re'] = re.compile(hdict['pattern']) diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py index c0c928a45..fbfab1ade 100644 --- a/module/plugins/internal/MultiHoster.py +++ b/module/plugins/internal/MultiHoster.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class MultiHoster(SimpleHoster): __name__ = "MultiHoster" __type__ = "hoster" - __version__ = "0.50" + __version__ = "0.51" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -21,21 +21,21 @@ class MultiHoster(SimpleHoster): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - HOSTER_NAME = None + PLUGIN_NAME = None LEECH_HOSTER = False LOGIN_ACCOUNT = True def init(self): - self.HOSTER_NAME = self.pyload.pluginManager.hosterPlugins[self.__name__]['name'] + self.PLUGIN_NAME = self.pyload.pluginManager.hosterPlugins[self.__name__]['name'] def _log(self, level, plugintype, pluginname, messages): return super(MultiHoster, self)._log(level, plugintype, pluginname, - (self.HOSTER_NAME,) + messages) + (self.PLUGIN_NAME,) + messages) def setup(self): @@ -83,11 +83,11 @@ class MultiHoster(SimpleHoster): self.check_errors() self.check_status(getinfo=False) - if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): + if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as premium download...")) self.handle_premium(pyfile) - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as free download...")) self.handle_free(pyfile) @@ -95,7 +95,7 @@ class MultiHoster(SimpleHoster): self.log_info(_("Downloading file...")) self.download(self.link, disposition=self.DISPOSITION) - self.check_file() + self.check_download() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 if self.premium: diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 51192d8c9..f1cd74fcb 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -6,7 +6,6 @@ import datetime import inspect import os import re -import sys import traceback import urllib import urlparse @@ -28,6 +27,11 @@ def decode(string, encoding='utf8'): return unicode(string) +#@TODO: Remove in 0.4.10 +def _decode(*args, **kwargs): + return decode(*args, **kwargs) + + #@TODO: Move to utils in 0.4.10 def encode(string, encoding='utf8'): """ Decode string to utf8 """ @@ -49,18 +53,25 @@ def exists(path): return False -#@TODO: Move to utils in 0.4.10 -def parse_name(url): - url = urllib.unquote(url) - url = url.decode('unicode-escape') - url = html_unescape(url) - url = urllib.quote(url) +#@TODO: Recheck in 0.4.10 +def fixurl(url, unquote=None): + newurl = urllib.unquote(url) + + if unquote is None: + unquote = newurl == url + + newurl = html_unescape(newurl.decode('unicode-escape')).strip() - url_p = urlparse.urlparse(url.strip().rstrip('/')) + return newurl if unquote else urllib.quote(newurl) - name = (url_p.path.split('/')[-1] or - url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or - url_p.netloc.split('.', 1)[0]) + +#@TODO: Recheck in 0.4.10 +def parse_name(string): + path = fixurl(decode(string), unquote=False) + url_p = urlparse.urlparse(path.rstrip('/')) + name = (url_p.path.split('/')[-1] or + url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] or + url_p.netloc.split('.', 1)[0]) return urllib.unquote(name) @@ -96,7 +107,7 @@ def seconds_to_midnight(utc=None): else: now = datetime.datetime.utcnow() + datetime.timedelta(hours=utc) - midnight = now.replace(hour=0, minute=10, second=0, microsecond=0) + datetime.timedelta(days=1) + midnight = now.replace(hour=0, minute=1, second=0, microsecond=0) + datetime.timedelta(days=1) return (midnight - now).seconds @@ -173,7 +184,7 @@ def chunks(iterable, size): class Plugin(object): __name__ = "Plugin" __type__ = "plugin" - __version__ = "0.37" + __version__ = "0.38" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -199,8 +210,8 @@ class Plugin(object): def _init(self, core): self.pyload = core - self.info = {} #: Provide information in dict here - self.req = None + self.info = {} #: Provide information in dict here + self.req = None #: Browser instance, see `network.Browser` def init(self): @@ -231,8 +242,6 @@ class Plugin(object): def log_warning(self, *args): self._log("warning", self.__type__, self.__name__, args) - if self.pyload.debug: - traceback.print_exc() def log_error(self, *args): @@ -278,7 +287,7 @@ class Plugin(object): return min(self.pyload.config.get("download", "chunks"), self.chunk_limit) - def set_config(self, option, value): + def set_config(self, option, value, plugin=None): """ Set config value for current plugin @@ -286,7 +295,7 @@ class Plugin(object): :param value: :return: """ - self.pyload.config.setPlugin(self.__name__, option, value) + self.pyload.config.setPlugin(plugin or self.__name__, option, value) def get_config(self, option, default="", plugin=None): @@ -347,7 +356,9 @@ class Plugin(object): """ if self.pyload.debug: self.log_debug("LOAD URL " + url, - *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url")]) + *["%s=%s" % (key, val) for key, val in locals().items() if key not in ("self", "url", "_[1]")]) + + url = fixurl(url, unquote=True) #: Recheck in 0.4.10 if req is None: req = self.req or self.pyload.requestFactory.getRequest(self.__name__) @@ -364,7 +375,7 @@ class Plugin(object): #@TODO: Move to network in 0.4.10 if isinstance(decode, basestring): - res = sys.modules[self.__name__].decode(res, decode) #@TODO: See #1787, use utils.decode() in 0.4.10 + res = _decode(res, decode) #@NOTE: Use `utils.decode()` in 0.4.10 if self.pyload.debug: frame = inspect.currentframe() @@ -409,11 +420,12 @@ class Plugin(object): Clean everything and remove references """ try: + self.req.clearCookies() self.req.close() except Exception: pass - for a in ("pyfile", "thread", "html", "req"): - if hasattr(self, a): - setattr(self, a, None) + for attr in ("account", "html", "pyfile", "req", "thread"): + if hasattr(self, attr): + setattr(self, attr, None) diff --git a/module/plugins/internal/SevenZip.py b/module/plugins/internal/SevenZip.py index b79256536..9d3a71a21 100644 --- a/module/plugins/internal/SevenZip.py +++ b/module/plugins/internal/SevenZip.py @@ -5,12 +5,12 @@ import re import subprocess from module.plugins.internal.UnRar import ArchiveError, CRCError, PasswordError, UnRar, renice -from module.utils import fs_encode, save_join as fs_join +from module.utils import save_join as fs_join class SevenZip(UnRar): __name__ = "SevenZip" - __version__ = "0.15" + __version__ = "0.16" __status__ = "testing" __description__ = """7-Zip extractor plugin""" @@ -128,7 +128,7 @@ class SevenZip(UnRar): args.append("-y") #: Set a password - if "password" in kwargs and kwargs['password']: + if kwargs.get('password'): args.append("-p%s" % kwargs['password']) else: args.append("-p-") diff --git a/module/plugins/internal/SimpleCrypter.py b/module/plugins/internal/SimpleCrypter.py index 8c5d3599d..53d69b2a5 100644 --- a/module/plugins/internal/SimpleCrypter.py +++ b/module/plugins/internal/SimpleCrypter.py @@ -9,7 +9,7 @@ from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, r class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.62" + __version__ = "0.63" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -81,7 +81,7 @@ class SimpleCrypter(Crypter, SimpleHoster): self.log_debug("Redirect #%d to: %s" % (i, redirect)) header = self.load(redirect, just_header=True) - if 'location' in header and header['location']: + if header.get('location'): self.link = header['location'] else: break @@ -137,7 +137,7 @@ class SimpleCrypter(Crypter, SimpleHoster): try: pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) - except AttributeError: + except (AttributeError, IndexError, ValueError): pages = 1 for p in xrange(2, pages + 1): diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 0f030e000..00b63ac12 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -6,7 +6,6 @@ import os import re import time -from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo @@ -14,14 +13,10 @@ from module.plugins.internal.Plugin import Fail, encode, parse_name, replace_pat from module.utils import fixup, fs_encode, parseFileSize as parse_size -#@TODO: Adapt and move to PyFile in 0.4.10 -statusMap = dict((v, k) for k, v in _statusMap.items()) - - class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "1.86" + __version__ = "1.87" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -124,7 +119,7 @@ class SimpleHoster(Hoster): try: info['pattern'] = re.match(cls.__pattern__, url).groupdict() #: Pattern groups will be saved here - except AttributeError: + except (AttributeError, IndexError): info['pattern'] = {} if not html and not online: @@ -164,7 +159,7 @@ class SimpleHoster(Hoster): if all(True for k in pdict if k not in info['pattern']): info['pattern'].update(pdict) - except AttributeError: + except (AttributeError, IndexError): continue else: @@ -275,11 +270,11 @@ class SimpleHoster(Hoster): if 'status' not in self.info or self.info['status'] is 3: #@TODO: Recheck in 0.4.10 self.check_info() - if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic_left()): + if self.premium and (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as premium download...")) self.handle_premium(pyfile) - elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic_left()): + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.check_traffic()): self.log_info(_("Processing as free download...")) self.handle_free(pyfile) @@ -287,7 +282,7 @@ class SimpleHoster(Hoster): self.log_info(_("Downloading file...")) self.download(self.link, disposition=self.DISPOSITION) - self.check_file() + self.check_download() except Fail, e: #@TODO: Move to PluginThread in 0.4.10 if self.get_config('premium_fallback', True) and self.premium: @@ -298,26 +293,26 @@ class SimpleHoster(Hoster): raise Fail(encode(e)) #@TODO: Remove `encode` in 0.4.10 - def check_file(self): - self.log_info(_("Checking file...")) + def check_download(self): + self.log_info(_("Checking downloaded file...")) if self.captcha.task and not self.last_download: self.captcha.invalid() self.retry(10, msg=_("Wrong captcha")) - elif self.check_download({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, + elif self.check_file({'Empty file': re.compile(r'\A((.|)(\2|\s)*)\Z')}, delete=True): self.error(_("Empty file")) else: - if self.get_config('chk_filesize', False) and 'size' in self.info: + if self.get_config('chk_filesize', False) and self.info.get('size'): # 10485760 is 10MB, tolerance is used when comparing displayed size on the hoster website to real size # For example displayed size can be 1.46GB for example, but real size can be 1.4649853GB self.check_filesize(self.info['size'], size_tolerance=10485760) self.log_debug("Using default check rules...") for r, p in self.FILE_ERRORS: - errmsg = self.check_download({r: re.compile(p)}) + errmsg = self.check_file({r: re.compile(p)}) if errmsg is not None: errmsg = errmsg.strip().capitalize() @@ -338,7 +333,6 @@ class SimpleHoster(Hoster): self.check_errors() self.log_info(_("No errors found")) - self.pyfile.error = "" #@TODO: Recheck in 0.4.10 def check_errors(self): @@ -361,7 +355,7 @@ class SimpleHoster(Hoster): try: errmsg = m.group(1).strip() - except AttributeError: + except (AttributeError, IndexError): errmsg = m.group(0).strip() self.info['error'] = re.sub(r'<.*?>', " ", errmsg) @@ -385,7 +379,7 @@ class SimpleHoster(Hoster): try: errmsg = m.group(1).strip() - except AttributeError: + except (AttributeError, IndexError): errmsg = m.group(0).strip() self.info['error'] = re.sub(r'<.*?>', " ", errmsg) @@ -436,7 +430,7 @@ class SimpleHoster(Hoster): try: waitmsg = m.group(1).strip() - except AttributeError: + except (AttributeError, IndexError): waitmsg = m.group(0).strip() wait_time = sum(int(v) * {'hr': 3600, 'hour': 3600, 'min': 60, 'sec': 1, "": 1}[u.lower()] for v, u in @@ -455,22 +449,19 @@ class SimpleHoster(Hoster): self.log_debug("Previous file info: %s" % old_info) try: - status = self.info['status'] or None + status = self.info['status'] or 14 - if status == 1: + if status is 1: self.offline() - elif status == 6: + elif status is 6: self.temp_offline() - elif status == 8: - if 'error' in self.info: - self.fail(self.info['error']) - else: - self.fail(_("File status: " + statusMap[status])) + elif status is 8: + self.fail() finally: - self.log_info(_("File status: ") + (statusMap[status] if status else _("Unknown"))) + self.log_info(_("File status: ") + self.pyfile.getStatusName(status)) def check_name_size(self, getinfo=True): @@ -492,7 +483,7 @@ class SimpleHoster(Hoster): if name and name is not url: self.pyfile.name = name - if 'size' in self.info and self.info['size'] > 0: + if self.info.get('size') > 0: self.pyfile.size = int(self.info['size']) #@TODO: Fix int conversion in 0.4.10 # self.pyfile.sync() diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 88c490750..7efecc5ef 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -8,7 +8,7 @@ from glob import glob from string import digits from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -from module.utils import fs_decode, fs_encode, save_join as fs_join +from module.utils import fs_decode, save_join as fs_join def renice(pid, value): @@ -22,7 +22,7 @@ def renice(pid, value): class UnRar(Extractor): __name__ = "UnRar" - __version__ = "1.26" + __version__ = "1.27" __status__ = "testing" __description__ = """Rar extractor plugin""" @@ -229,7 +229,7 @@ class UnRar(Extractor): args.append("-y") #: Set a password - if "password" in kwargs and kwargs['password']: + if kwargs.get('password'): args.append("-p%s" % kwargs['password']) else: args.append("-p-") diff --git a/module/plugins/internal/UnZip.py b/module/plugins/internal/UnZip.py index ac197a80d..87cbd568a 100644 --- a/module/plugins/internal/UnZip.py +++ b/module/plugins/internal/UnZip.py @@ -7,7 +7,6 @@ import sys import zipfile from module.plugins.internal.Extractor import Extractor, ArchiveError, CRCError, PasswordError -from module.utils import fs_encode class UnZip(Extractor): diff --git a/module/plugins/internal/XFSCrypter.py b/module/plugins/internal/XFSCrypter.py index 12a48d4a3..8047fd03e 100644 --- a/module/plugins/internal/XFSCrypter.py +++ b/module/plugins/internal/XFSCrypter.py @@ -7,7 +7,7 @@ from module.plugins.internal.SimpleCrypter import SimpleCrypter, create_getInfo class XFSCrypter(SimpleCrypter): __name__ = "XFSCrypter" __type__ = "crypter" - __version__ = "0.14" + __version__ = "0.15" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -17,7 +17,7 @@ class XFSCrypter(SimpleCrypter): __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] - HOSTER_DOMAIN = None + PLUGIN_DOMAIN = None URL_REPLACEMENTS = [(r'&?per_page=\d+', ""), (r'[?/&]+$', ""), (r'(.+/[^?]+)$', r'\1?'), (r'$', r'&per_page=10000')] @@ -29,22 +29,22 @@ class XFSCrypter(SimpleCrypter): def prepare(self): - if not self.HOSTER_DOMAIN: + if not self.PLUGIN_DOMAIN: if self.account: account = self.account else: account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") account = self.pyload.accountManager.getAccountPlugin(account_name) - if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: - self.HOSTER_DOMAIN = account.HOSTER_DOMAIN + if account and hasattr(account, "PLUGIN_DOMAIN") and account.PLUGIN_DOMAIN: + self.PLUGIN_DOMAIN = account.PLUGIN_DOMAIN else: - self.fail(_("Missing HOSTER_DOMAIN")) + self.fail(_("Missing PLUGIN_DOMAIN")) if self.COOKIES: - if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: + self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) else: - set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") + set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") return super(XFSCrypter, self).prepare() diff --git a/module/plugins/internal/XFSHoster.py b/module/plugins/internal/XFSHoster.py index 729c9a0ee..8af6b83ed 100644 --- a/module/plugins/internal/XFSHoster.py +++ b/module/plugins/internal/XFSHoster.py @@ -14,7 +14,7 @@ from module.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.60" + __version__ = "0.61" __status__ = "testing" __pattern__ = r'^unmatchable$' @@ -26,7 +26,7 @@ class XFSHoster(SimpleHoster): ("Walter Purcaro", "vuolter@gmail.com" )] - HOSTER_DOMAIN = None + PLUGIN_DOMAIN = None LEECH_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... @@ -61,26 +61,26 @@ class XFSHoster(SimpleHoster): """ Initialize important variables """ - if not self.HOSTER_DOMAIN: + if not self.PLUGIN_DOMAIN: if self.account: account = self.account else: account = self.pyload.accountManager.getAccountPlugin(self.__name__) - if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: - self.HOSTER_DOMAIN = account.HOSTER_DOMAIN + if account and hasattr(account, "PLUGIN_DOMAIN") and account.PLUGIN_DOMAIN: + self.PLUGIN_DOMAIN = account.PLUGIN_DOMAIN else: - self.fail(_("Missing HOSTER_DOMAIN")) + self.fail(_("Missing PLUGIN_DOMAIN")) if self.COOKIES: - if isinstance(self.COOKIES, list) and (self.HOSTER_DOMAIN, "lang", "english") not in self.COOKIES: - self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + if isinstance(self.COOKIES, list) and (self.PLUGIN_DOMAIN, "lang", "english") not in self.COOKIES: + self.COOKIES.insert((self.PLUGIN_DOMAIN, "lang", "english")) else: - set_cookie(self.req.cj, self.HOSTER_DOMAIN, "lang", "english") + set_cookie(self.req.cj, self.PLUGIN_DOMAIN, "lang", "english") if not self.LINK_PATTERN: pattern = r'(?:file: "(.+?)"|(https?://(?:www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<])' - self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.') + self.LINK_PATTERN = pattern % self.PLUGIN_DOMAIN.replace('.', '\.') super(XFSHoster, self).prepare() @@ -129,7 +129,7 @@ class XFSHoster(SimpleHoster): self.fail(_("Only registered or premium users can use url leech feature")) #: Only tested with easybytez.com - self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) + self.html = self.load("http://www.%s/" % self.PLUGIN_DOMAIN) action, inputs = self.parse_html_form() @@ -244,7 +244,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) - except AttributeError: + except (AttributeError, IndexError): captcha_key = recaptcha.detect_key() else: @@ -258,7 +258,7 @@ class XFSHoster(SimpleHoster): try: captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) - except AttributeError: + except (AttributeError, IndexError): captcha_key = solvemedia.detect_key() else: -- cgit v1.2.3