diff options
Diffstat (limited to 'module/plugins/internal')
| -rw-r--r-- | module/plugins/internal/Account.py | 95 | ||||
| -rw-r--r-- | module/plugins/internal/Addon.py | 2 | ||||
| -rw-r--r-- | module/plugins/internal/Captcha.py | 5 | ||||
| -rw-r--r-- | module/plugins/internal/CaptchaService.py | 8 | ||||
| -rw-r--r-- | module/plugins/internal/Container.py | 3 | ||||
| -rw-r--r-- | module/plugins/internal/Hoster.py | 36 | ||||
| -rw-r--r-- | module/plugins/internal/Plugin.py | 15 | ||||
| -rw-r--r-- | module/plugins/internal/SimpleHoster.py | 4 |
8 files changed, 102 insertions, 66 deletions
diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py index 3a3fd4768..f33d44179 100644 --- a/module/plugins/internal/Account.py +++ b/module/plugins/internal/Account.py @@ -4,6 +4,7 @@ from __future__ import with_statement import random import time +import threading import traceback from module.plugins.internal.Plugin import Plugin @@ -29,6 +30,7 @@ class Account(Plugin): def __init__(self, manager, accounts): self.pyload = manager.core self.info = {} #: Provide information in dict here + self.lock = threading.RLock() self.init() self.init_accounts(accounts) @@ -58,9 +60,7 @@ class Account(Plugin): self.login(user, info['login']['password'], info['data'], self.req) except Exception, e: - self.log_warning( - _("Could not login with account %(user)s | %(msg)s") % {'user': user, - 'msg' : e}) + self.log_warning(_("Could not login with username ") + user, e) res = info['login']['valid'] = False if self.pyload.debug: traceback.print_exc() @@ -86,13 +86,15 @@ class Account(Plugin): return self._login(user) + #@TODO: Rewrite in 0.4.10 def init_accounts(self, accounts): for user, data in accounts.iteritems(): self.add(user, data['password'], data['options']) self._login(user) - def add(self, user, password=None, options): + @lock + def add(self, user, password=None, options={}): if user not in self.info: self.info[user] = {'login': {'valid': True, 'password': password or "", 'timestamp': 0}, #@NOTE: Do not remove `'valid': True` in 0.4.9 or accounts will not login 'data' : {'options': options, 'timestamp': 0}} @@ -102,7 +104,8 @@ class Account(Plugin): self.log_error(_("Error adding account"), _("User already exists")) - def update(self, user, password=None, options): + @lock + def update(self, user, password=None, options={}): """ Updates account and return true if anything changed """ @@ -131,7 +134,7 @@ class Account(Plugin): self.info.clear() elif user in self.info: - del self.info[user] + self.info.pop(user, None) #: Deprecated method, use `remove` instead (Remove in 0.4.10) @@ -140,64 +143,88 @@ class Account(Plugin): def get_data(self, user, reload=False): - return self.get_info(user, reload)['data'] + if not user: + return + + info = self.get_info(user, reload) + if info and 'data' in info: + return info['data'] + + #: Deprecated method, use `get_data` instead (Remove in 0.4.10) + def getAccountData(self, *args, **kwargs): + if 'force' in kwargs: + kwargs['reload'] = kwargs['force'] + kwargs.pop('force', None) + data = self.get_data(*args, **kwargs) or {} + if 'options' not in data: + data['options'] = {'limitdl': ['0']} + return data @lock def get_info(self, user, reload=False): """ Retrieve account infos for an user, do **not** overwrite this method!\\ - just use it to retrieve infos in hoster plugins. see `load_info` + just use it to retrieve infos in hoster plugins. see `parse_info` :param user: username :param reload: reloads cached account information :return: dictionary with information """ - if not reload and user in self.info: - info = self.info[user] + traceback.print_exc() ###################### - if info['data']['timestamp'] + self.INFO_THRESHOLD * 60 < time.time(): - self.log_debug("Reached timeout for account data") - self.schedule_refresh(user) + if user not in self.info: + self.log_error(_("User %s not found while retrieving account info") % user) + return - else: + elif reload: self.log_debug("Get Account Info for: %s" % user) - info = self.load_info(user) + info = self._parse_info(user) - self.log_debug("Account Info: %s" % info) + else: + info = self.info[user] + + if self.INFO_THRESHOLD > 0 and info['data']['timestamp'] + self.INFO_THRESHOLD * 60 < time.time(): + self.log_debug("Reached data timeout for %s" % user) + self.schedule_refresh(user) + + safe_info = info.copy() + safe_info['login']['password'] = "**********" + self.log_debug("Account info: %s" % safe_info) return info def is_premium(self, user): - return self.get_info(user)['premium'] + if not user: + return False + info = self.get_info(user, reload) + return info['premium'] if info and 'premium' in info else False - def load_info(self, user): - self.log_critical(user in self.info) ############################# + def _parse_info(self, user): info = self.info[user] data = info['data'] #@TODO: Remove in 0.4.10 data.update({'login': user, - 'type' : self.__name__}, - 'valid': self.info[user]['login']['valid']) + 'type' : self.__name__, + 'valid': self.info[user]['login']['valid']}) try: + data['timestamp'] = time.time() #: Set timestamp for login + self.req = self.get_request(user) - extra_info = self.parse_info(user, info['login']['password'], info, req) + extra_info = self.parse_info(user, info['login']['password'], info, self.req) if extra_info and isinstance(extra_info, dict): data.update(extra_info) - data['timestamp'] = time.time() except Exception, e: - self.log_warning(_("Error loading info for account %(user)s | %(err)s") % - {'user': user, 'err': e}) - data['error'] = str(e) + self.log_warning(_("Error loading info for ") + user, e) if self.pyload.debug: - traceback.print_exc(): + traceback.print_exc() else: for key in ('premium', 'validuntil', 'trafficleft', 'maxtraffic'): @@ -227,7 +254,11 @@ class Account(Plugin): #: Remove in 0.4.10 def getAllAccounts(self, *args, **kwargs): - return [self.get_data(user, reload) for user, info in self.info.iteritems()] + return [self.getAccountData(user, *args, **kwargs) for user, info in self.info.iteritems()] + + + def login_fail(self, reason=_("Login handshake has failed")): + return self.fail(reason) def get_request(self, user=None): @@ -240,7 +271,6 @@ class Account(Plugin): def get_cookies(self, user=None): if not user: user, info = self.select() - return None return self.pyload.requestFactory.getCookieJar(self.__name__, user) @@ -316,17 +346,20 @@ class Account(Plugin): """ Add task to refresh account info to sheduler """ - self.log_debug("Scheduled Account refresh for %s in %s seconds." % (user, time)) + self.log_debug("Scheduled refresh for %s in %s seconds" % (user, time)) self.pyload.scheduler.addJob(time, self.get_info, [user, reload]) #: Deprecated method, use `schedule_refresh` instead (Remove in 0.4.10) def scheduleRefresh(self, *args, **kwargs): + if 'force' in kwargs: + kwargs['reload'] = kwargs['force'] + kwargs.pop('force', None) return self.schedule_refresh(*args, **kwargs) @lock - def check_login(self, user): + def is_logged(self, user): """ Checks if user is still logged in """ diff --git a/module/plugins/internal/Addon.py b/module/plugins/internal/Addon.py index 16aab6188..248518128 100644 --- a/module/plugins/internal/Addon.py +++ b/module/plugins/internal/Addon.py @@ -71,7 +71,7 @@ class Addon(Plugin): self.event_map = None if self.event_list: - self.log_debug("Deprecated method `event_list`, use `event_map` instead") (Remove in 0.4.10) + self.log_debug("Deprecated method `event_list`, use `event_map` instead") for f in self.event_list: self.manager.addEvent(f, getattr(self, f)) diff --git a/module/plugins/internal/Captcha.py b/module/plugins/internal/Captcha.py index 89cace612..ec618e353 100644 --- a/module/plugins/internal/Captcha.py +++ b/module/plugins/internal/Captcha.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import os import time import traceback @@ -27,10 +28,6 @@ class Captcha(Plugin): self.init() - def _log(self, level, args): - return self.plugin._log(level, (self.__name__,) + args) - - def init(self): """ Initialize additional data structures diff --git a/module/plugins/internal/CaptchaService.py b/module/plugins/internal/CaptchaService.py index d3eef43c8..1a73ce5d1 100644 --- a/module/plugins/internal/CaptchaService.py +++ b/module/plugins/internal/CaptchaService.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -from module.plugins.internal.Plugin import Plugin +from module.plugins.internal.Captcha import Captcha -class CaptchaService(Plugin): +class CaptchaService(Captcha): __name__ = "CaptchaService" __type__ = "captcha" __version__ = "0.31" @@ -18,6 +18,10 @@ class CaptchaService(Plugin): self.key = None #: Last key detected + def _log(self, level, args): + return self.plugin._log(level, (self.__name__,) + args) + + #@TODO: Recheck in 0.4.10 def retrieve_key(self, data): if self.detect_key(data): diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py index c0135fb1f..c7a6828c7 100644 --- a/module/plugins/internal/Container.py +++ b/module/plugins/internal/Container.py @@ -56,8 +56,9 @@ class Container(Crypter): try: with open(self.pyfile.url, "wb") as f: f.write(content) + except IOError, e: - self.fail(str(e)) + self.fail(str(e)) #@TODO: Remove `str` in 0.4.10 else: self.pyfile.name = os.path.basename(self.pyfile.url) diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py index 1d2728bd3..840d73847 100644 --- a/module/plugins/internal/Hoster.py +++ b/module/plugins/internal/Hoster.py @@ -171,7 +171,7 @@ class Hoster(Plugin): self.req.renewHTTPRequest() if self.account: - self.account.check_login(self.user) + self.account.is_logged(self.user) else: self.req.clearCookies() @@ -206,7 +206,10 @@ class Hoster(Plugin): def set_reconnect(self, reconnect): reconnect = bool(reconnect) - self.log_debug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) + + self.log_info(_("Reconnect: %s") % reconnect) + self.log_debug("Previous wantReconnect: %s" % self.wantReconnect) + self.wantReconnect = reconnect @@ -220,8 +223,8 @@ class Hoster(Plugin): wait_time = max(int(seconds), 1) wait_until = time.time() + wait_time + 1 - self.log_debug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), - "Wait: %d (+1) seconds" % wait_time) + self.log_info(_("Wait: %d seconds") % wait_time) + self.log_debug("Previous waitUntil: %f" % self.pyfile.waitUntil) self.pyfile.waitUntil = wait_until @@ -229,13 +232,13 @@ class Hoster(Plugin): self.set_reconnect(reconnect) - def wait(self, seconds=0, reconnect=None): + def wait(self, seconds=None, reconnect=None): """ Waits the time previously set """ pyfile = self.pyfile - if seconds > 0: + if seconds is not None: self.set_wait(seconds) if reconnect is not None: @@ -243,12 +246,9 @@ class Hoster(Plugin): self.waiting = True - status = pyfile.status + status = pyfile.status #@NOTE: Remove in 0.4.10 pyfile.setStatus("waiting") - self.log_info(_("Wait: %d seconds") % (pyfile.waitUntil - time.time()), - _("Reconnect: %s") % self.wantReconnect) - if self.account: self.log_debug("Ignore reconnection due account logged") @@ -256,11 +256,9 @@ class Hoster(Plugin): if pyfile.abort: self.abort() - time.sleep(1) + time.sleep(3) else: while pyfile.waitUntil > time.time(): - self.thread.m.reconnecting.wait(1) - if pyfile.abort: self.abort() @@ -268,12 +266,12 @@ class Hoster(Plugin): self.waiting = False self.wantReconnect = False raise Reconnect - - time.sleep(1) + + self.thread.m.reconnecting.wait(3) + time.sleep(3) self.waiting = False - - pyfile.status = status + pyfile.status = status #@NOTE: Remove in 0.4.10 def skip(self, reason=""): @@ -388,7 +386,7 @@ class Hoster(Plugin): os.chown(location, uid, gid) except Exception, e: - self.fail(e) + self.fail(str(e)) #@TODO: Remove `str` in 0.4.10 #: Convert back to unicode location = fs_decode(location) @@ -508,7 +506,7 @@ class Hoster(Plugin): redirect = max(follow_location, 1) else: - redirect = self.get_config("maxredirs", plugin="UserAgentSwitcher") + redirect = self.get_config("maxredirs", 10, "UserAgentSwitcher") for i in xrange(redirect): try: diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py index 9ac89501c..02ef326d6 100644 --- a/module/plugins/internal/Plugin.py +++ b/module/plugins/internal/Plugin.py @@ -5,6 +5,7 @@ from __future__ import with_statement import inspect import os import re +import traceback import urllib from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload as Skip #@TODO: Remove in 0.4.10 @@ -80,12 +81,12 @@ def parse_html_form(attr_str, html, input_names={}): for form in re.finditer(r"(?P<TAG><form[^>]*%s[^>]*>)(?P<CONTENT>.*?)</?(form|body|html)[^>]*>" % attr_str, html, re.S | re.I): inputs = {} - action = parseHtmlTagAttrValue("action", form.group('TAG')) + action = parse_html_tag_attr_value("action", form.group('TAG')) for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=</\2)|)', form.group('CONTENT'), re.S | re.I): - name = parseHtmlTagAttrValue("name", inputtag.group(1)) + name = parse_html_tag_attr_value("name", inputtag.group(1)) if name: - value = parseHtmlTagAttrValue("value", inputtag.group(1)) + value = parse_html_tag_attr_value("value", inputtag.group(1)) if not value: inputs[name] = inputtag.group(3) or "" else: @@ -155,9 +156,10 @@ class Plugin(object): def _log(self, level, args): log = getattr(self.pyload.log, level) msg = encode(" | ".join((a if isinstance(a, basestring) else str(a)).strip() for a in args if a)) #@NOTE: `fs_encode` -> `encode` in 0.4.10 - log("%(plugin)s%(id)s: %(msg)s" % {'plugin': self.__name__, - 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", - 'msg' : msg or _(level.upper() + " MARK")}) + log("%(type)s %(plugin)s%(id)s: %(msg)s" % {'type': self.__type__.upper(), + 'plugin': self.__name__, + 'id' : ("[%s]" % self.pyfile.id) if hasattr(self, 'pyfile') else "", + 'msg' : msg or _(level.upper() + " MARK")}) def log_debug(self, *args): @@ -204,6 +206,7 @@ class Plugin(object): except KeyError: self.log_warning(_("Config option or plugin not found")) + traceback.print_exc() return default diff --git a/module/plugins/internal/SimpleHoster.py b/module/plugins/internal/SimpleHoster.py index 4a293b703..8eea9dd5a 100644 --- a/module/plugins/internal/SimpleHoster.py +++ b/module/plugins/internal/SimpleHoster.py @@ -13,7 +13,7 @@ from module.PyFile import statusMap as _statusMap from module.network.HTTPRequest import BadHeader from module.network.RequestFactory import getURL as get_url from module.plugins.internal.Hoster import Hoster, create_getInfo, parse_fileInfo -from module.plugins.internal.Plugin import Fail, Retry, fixurl, replace_patterns, set_cookies +from module.plugins.internal.Plugin import Fail, Retry, fixurl, replace_patterns, seconds_to_midnight, set_cookies from module.utils import fixup, fs_encode, parseFileSize as parse_size @@ -115,7 +115,7 @@ class SimpleHoster(Hoster): @classmethod def api_info(cls, url): - return super(SimpleHoster, self).get_info(url) + return super(SimpleHoster, cls).get_info(url) @classmethod |