From 7b8c458cca7d21a029620f98e453f746fce69cd1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 14 Jul 2014 16:10:01 +0200 Subject: Prefer single quote for dict key name --- module/plugins/Plugin.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 584fcce49..8722496b5 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -207,8 +207,8 @@ class Plugin(Base): def getChunkCount(self): if self.chunkLimit <= 0: - return self.config["download"]["chunks"] - return min(self.config["download"]["chunks"], self.chunkLimit) + return self.config['download']['chunks'] + return min(self.config['download']['chunks'], self.chunkLimit) def __call__(self): return self.__name__ @@ -478,12 +478,12 @@ class Plugin(Base): location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): - makedirs(location, int(self.core.config["permission"]["folder"], 8)) + makedirs(location, int(self.core.config['permission']['folder'], 8)) - if self.core.config["permission"]["change_dl"] and os.name != "nt": + if self.core.config['permission']['change_dl'] and os.name != "nt": try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] + uid = getpwnam(self.config['permission']['user'])[2] + gid = getgrnam(self.config['permission']['group'])[2] chown(location, uid, gid) except Exception, e: @@ -511,13 +511,13 @@ class Plugin(Base): fs_filename = fs_encode(filename) - if self.core.config["permission"]["change_file"]: - chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) + if self.core.config['permission']['change_file']: + chmod(fs_filename, int(self.core.config['permission']['file'], 8)) - if self.core.config["permission"]["change_dl"] and os.name != "nt": + if self.core.config['permission']['change_dl'] and os.name != "nt": try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] + uid = getpwnam(self.config['permission']['user'])[2] + gid = getgrnam(self.config['permission']['group'])[2] chown(fs_filename, uid, gid) except Exception, e: -- cgit v1.2.3 From ba916633f2bedb04c7358000b91aed69f52e8e43 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 1 Aug 2014 19:35:59 +0200 Subject: Remove trailing whitespaces + remove license headers + import urllib methods directly + sort and fix key attributes + use save_join instead join + sort some import declarations + other minor code cosmetics --- module/plugins/Plugin.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 8722496b5..68b2311b3 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -1,22 +1,5 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN, spoob, mkaay -""" - from time import time, sleep from random import randint @@ -33,6 +16,7 @@ from itertools import islice from module.utils import save_join, save_path, fs_encode, fs_decode + def chunks(iterable, size): it = iter(iterable) item = list(islice(it, size)) @@ -142,14 +126,17 @@ class Plugin(Base): Overwrite `process` / `decrypt` in your subclassed plugin. """ __name__ = "Plugin" + __type__ = "hoster" __version__ = "0.4" + __pattern__ = None - __type__ = "hoster" __config__ = [("name", "type", "desc", "default")] + __description__ = """Base plugin""" __author_name__ = ("RaNaN", "spoob", "mkaay") __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") + def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) -- cgit v1.2.3 From 8add4efc7dc3eeaa873030930e5bd31d6bdf8126 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 28 Sep 2014 14:57:02 +0200 Subject: [XFileSharingPro] COOKIES preset to english + improved setup routine --- module/plugins/Plugin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 68b2311b3..bbfd87532 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -397,7 +397,8 @@ class Plugin(Base): """ if self.pyfile.abort: raise Abort #utf8 vs decode -> please use decode attribute in all future plugins - if type(url) == unicode: url = str(url) + if type(url) == unicode: + url = str(url) # encode('utf8') res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) -- cgit v1.2.3 From b0868ae6446078bacf1635dde5e4ab316b4a94cb Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 7 Oct 2014 18:57:59 +0200 Subject: New __authors__ key replaces __author_name__ and __author_mail__ + Whitespaces and EOF fixup --- module/plugins/Plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index bbfd87532..70da88312 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -133,8 +133,9 @@ class Plugin(Base): __config__ = [("name", "type", "desc", "default")] __description__ = """Base plugin""" - __author_name__ = ("RaNaN", "spoob", "mkaay") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("spoob", "spoob@pyload.org"), + ("mkaay", "mkaay@mkaay.de")] def __init__(self, pyfile): -- cgit v1.2.3 From ae7a7e66981456e5bbe2b54006d79b6f907be7a4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 8 Oct 2014 20:18:13 +0200 Subject: Add __license__ key attribute to plugins --- module/plugins/Plugin.py | 1 + 1 file changed, 1 insertion(+) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 70da88312..fa1d2c3b1 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -133,6 +133,7 @@ class Plugin(Base): __config__ = [("name", "type", "desc", "default")] __description__ = """Base plugin""" + __license__ = "GPLv3" __authors__ = [("RaNaN", "RaNaN@pyload.org"), ("spoob", "spoob@pyload.org"), ("mkaay", "mkaay@mkaay.de")] -- cgit v1.2.3 From 18836967d39d0b6e6f2aeea4e6aece605246a2bf Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 17 Oct 2014 20:55:00 +0200 Subject: Spare code cosmetics --- module/plugins/Plugin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index fa1d2c3b1..d17776485 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -335,9 +335,9 @@ class Plugin(Base): img = self.load(url, get=get, post=post, cookies=cookies) id = ("%.2f" % time())[-6:].replace(".", "") - temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - temp_file.write(img) - temp_file.close() + tmpCaptcha = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + tmpCaptcha.write(img) + tmpCaptcha.close() has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins @@ -351,10 +351,10 @@ class Plugin(Base): if self.pyfile.abort: raise Abort ocr = Ocr() - result = ocr.get_captcha(temp_file.name) + result = ocr.get_captcha(tmpCaptcha.name) else: captchaManager = self.core.captchaManager - task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) + task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) self.cTask = task captchaManager.handleCaptcha(task) @@ -378,7 +378,7 @@ class Plugin(Base): if not self.core.debug: try: - remove(temp_file.name) + remove(tmpCaptcha.name) except: pass -- cgit v1.2.3 From 52ec86c18b649676b3231aa3730583041c132bd7 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 20 Oct 2014 00:32:04 +0200 Subject: Improve log functions --- module/plugins/Plugin.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index d17776485..57da4d114 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -59,17 +59,20 @@ class Base(object): self.config = core.config #log functions + def logDebug(self, *args): + self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) + def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) - def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + def logCritical(self, *args): + self.log.critical("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) def setConf(self, option, value): -- cgit v1.2.3 From 6badd4f1f210d2e9385c4d73c5b41ee5a6f04e65 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 09:59:16 +0200 Subject: Improve error method + fix TEXT_ENCODING in SimpleCrypter --- module/plugins/Plugin.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 57da4d114..83571345c 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -13,6 +13,7 @@ if os.name != "nt": from grp import getgrnam from itertools import islice +from traceback import print_exc from module.utils import save_join, save_path, fs_encode, fs_decode @@ -284,6 +285,11 @@ class Plugin(Base): """ fail and give reason """ raise Fail(reason) + def error(self, reason=None, type="parse"): + raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ':' + str(reason) if reason else "")) + if self.core.debug: + print_exc() + def offline(self): """ fail and indicate file is offline """ raise Fail("offline") -- cgit v1.2.3 From 1b096b2eb2634e8dea80b06ab9ecde206b198b35 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 19:47:00 +0200 Subject: Spare code cosmetics --- module/plugins/Plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 83571345c..0a4b6d44d 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -286,7 +286,7 @@ class Plugin(Base): raise Fail(reason) def error(self, reason=None, type="parse"): - raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ':' + str(reason) if reason else "")) + raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ': ' + str(reason) if reason else "")) if self.core.debug: print_exc() -- cgit v1.2.3 From 0eb6e7ec4a1144dcca824d8add049787d3da1762 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 22 Oct 2014 19:44:59 +0200 Subject: Two space before function declaration --- module/plugins/Plugin.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 0a4b6d44d..be26a8960 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -51,6 +51,7 @@ class Base(object): A Base class with log/config/db methods *all* plugin types can use """ + def __init__(self, core): #: Core instance self.core = core @@ -59,19 +60,24 @@ class Base(object): #: core config self.config = core.config + #log functions def logDebug(self, *args): self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) + def logInfo(self, *args): self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) + def logWarning(self, *args): self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) + def logError(self, *args): self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) + def logCritical(self, *args): self.log.critical("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) @@ -80,6 +86,7 @@ class Base(object): """ see `setConfig` """ self.core.config.setPlugin(self.__name__, option, value) + def setConfig(self, option, value): """ Set config value for current plugin @@ -89,10 +96,12 @@ class Base(object): """ self.setConf(option, value) + def getConf(self, option): """ see `getConfig` """ return self.core.config.getPlugin(self.__name__, option) + def getConfig(self, option): """ Returns config value for current plugin @@ -101,24 +110,29 @@ class Base(object): """ return self.getConf(option) + def setStorage(self, key, value): """ Saves a value persistently to the database """ self.core.db.setStorage(self.__name__, key, value) + def store(self, key, value): """ same as `setStorage` """ self.core.db.setStorage(self.__name__, key, value) + def getStorage(self, key=None, default=None): """ Retrieves saved value or dict of all saved entries if key is None """ if key is not None: return self.core.db.getStorage(self.__name__, key) or default return self.core.db.getStorage(self.__name__, key) + def retrieve(self, *args, **kwargs): """ same as `getStorage` """ return self.getStorage(*args, **kwargs) + def delStorage(self, key): """ Delete entry in db """ self.core.db.delStorage(self.__name__, key) @@ -198,22 +212,27 @@ class Plugin(Base): self.init() + def getChunkCount(self): if self.chunkLimit <= 0: return self.config['download']['chunks'] return min(self.config['download']['chunks'], self.chunkLimit) + def __call__(self): return self.__name__ + def init(self): """initialize the plugin (in addition to `__init__`)""" pass + def setup(self): """ setup for enviroment and other things, called before downloading (possibly more than one time)""" pass + def preprocessing(self, thread): """ handles important things to do before starting """ self.thread = thread @@ -234,12 +253,14 @@ class Plugin(Base): """the 'main' method of every plugin, you **have to** overwrite it""" raise NotImplementedError + def resetAccount(self): """ dont use account and retry download """ self.account = None self.req = self.core.requestFactory.getRequest(self.__name__) self.retry() + def checksum(self, local_file=None): """ return codes: @@ -264,6 +285,7 @@ class Plugin(Base): self.wantReconnect = True self.pyfile.waitUntil = time() + int(seconds) + def wait(self): """ waits the time previously set """ self.waiting = True @@ -281,23 +303,28 @@ class Plugin(Base): self.waiting = False self.pyfile.setStatus("starting") + def fail(self, reason): """ fail and give reason """ raise Fail(reason) + def error(self, reason=None, type="parse"): raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ': ' + str(reason) if reason else "")) if self.core.debug: print_exc() + def offline(self): """ fail and indicate file is offline """ raise Fail("offline") + def tempOffline(self): """ fail and indicates file ist temporary offline, the core may take consequences """ raise Fail("temp. offline") + def retry(self, max_tries=3, wait_time=1, reason=""): """Retries and begin again from the beginning @@ -316,14 +343,17 @@ class Plugin(Base): self.retries += 1 raise Retry(reason) + def invalidCaptcha(self): if self.cTask: self.cTask.invalid() + def correctCaptcha(self): if self.cTask: self.cTask.correct() + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', result_type='textual'): """ Loads a captcha and decrypts it with ocr, plugin, user input @@ -455,6 +485,7 @@ class Plugin(Base): return res + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder @@ -525,6 +556,7 @@ class Plugin(Base): self.lastDownload = filename return self.lastDownload + def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): """ checks the content of the last downloaded file, re match is saved to `lastCheck` @@ -603,6 +635,7 @@ class Plugin(Base): self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) + def clean(self): """ clean everything and remove references """ if hasattr(self, "pyfile"): -- cgit v1.2.3 From 8e89cb7781ec3b135feeb750f6321bbc85e89d67 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 23 Oct 2014 22:54:40 +0200 Subject: Fix error method --- module/plugins/Plugin.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index be26a8960..8e66dff6c 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -309,10 +309,17 @@ class Plugin(Base): raise Fail(reason) - def error(self, reason=None, type="parse"): - raise Fail("%s error%s | Plugin out of date" % (type.capitalize(), ': ' + str(reason) if reason else "")) + def error(self, reason="", type=""): + if not reason and not type: + type = "unknown" + + msg = "%s error" % type.strip().capitalize() if type else "Error" + msg += ": " + reason.strip() if reason else "" + msg += " | Plugin may be out of date" + if self.core.debug: print_exc() + raise Fail(msg) def offline(self): @@ -333,8 +340,7 @@ class Plugin(Base): :param reason: reason for retrying, will be passed to fail if max_tries reached """ if 0 < max_tries <= self.retries: - if not reason: reason = "Max retries reached" - raise Fail(reason) + self.error(reason or "Max retries reached", "retry") self.wantReconnect = False self.setWait(wait_time) -- cgit v1.2.3 From c41d4ec4c5935ee702a44e919730971028bf5724 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 25 Oct 2014 01:16:52 +0200 Subject: Fix and improve plugins logging --- module/plugins/Plugin.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 8e66dff6c..efb20ad59 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -63,23 +63,23 @@ class Base(object): #log functions def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) + self.log.debug("%s: %s" % (self.__name__, " | ".join([str(a).strip() for a in args if a]))) def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) + self.log.info("%s: %s" % (self.__name__, " | ".join([str(a).strip() for a in args if a]))) def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) + self.log.warning("%s: %s" % (self.__name__, " | ".join([str(a).strip() for a in args if a]))) def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) + self.log.error("%s: %s" % (self.__name__, " | ".join([str(a).strip() for a in args if a]))) def logCritical(self, *args): - self.log.critical("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args if a]))) + self.log.critical("%s: %s" % (self.__name__, " | ".join([str(a).strip() for a in args if a]))) def setConf(self, option, value): @@ -351,11 +351,13 @@ class Plugin(Base): def invalidCaptcha(self): + self.logError("Invalid captcha") if self.cTask: self.cTask.invalid() def correctCaptcha(self): + self.logInfo("Correct captcha") if self.cTask: self.cTask.correct() @@ -393,7 +395,8 @@ class Plugin(Base): if Ocr and not forceUser: sleep(randint(3000, 5000) / 1000.0) - if self.pyfile.abort: raise Abort + if self.pyfile.abort: + raise Abort ocr = Ocr() result = ocr.get_captcha(tmpCaptcha.name) @@ -419,7 +422,7 @@ class Plugin(Base): self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) result = task.result - self.log.debug("Received captcha result: %s" % str(result)) + self.logDebug("Received captcha result: %s" % str(result)) if not self.core.debug: try: @@ -447,6 +450,8 @@ class Plugin(Base): if type(url) == unicode: url = str(url) # encode('utf8') + self.logDebug("Load url", *["%s: %s" % (key, val) for key, val in locals().items()]) + res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.core.debug: @@ -505,6 +510,8 @@ class Plugin(Base): :return: The location where the file was saved """ + self.logDebug("Download url", *["%s: %s" % (key, val) for key, val in locals().items()]) + self.checkForSameFiles() self.pyfile.setStatus("downloading") @@ -523,7 +530,7 @@ class Plugin(Base): chown(location, uid, gid) except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) + self.logWarning(_("Setting User and Group failed: %s") % str(e)) # convert back to unicode location = fs_decode(location) @@ -541,7 +548,7 @@ class Plugin(Base): self.pyfile.size = self.req.size if disposition and newname and newname != name: #triple check, just to be sure - self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) + self.logInfo("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) self.pyfile.name = newname filename = join(location, newname) @@ -557,7 +564,7 @@ class Plugin(Base): chown(fs_filename, uid, gid) except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) + self.logWarning(_("Setting User and Group failed: %s") % str(e)) self.lastDownload = filename return self.lastDownload @@ -581,12 +588,12 @@ class Plugin(Base): if api_size and api_size <= size: return None elif size > max_size and not read_size: return None - self.log.debug("Download Check triggered") + self.logDebug("Download Check triggered") f = open(lastDownload, "rb") content = f.read(read_size if read_size else -1) f.close() #produces encoding errors, better log to other file in the future? - #self.log.debug("Content: %s" % content) + #self.logDebug("Content: %s" % content) for name, rule in rules.iteritems(): if type(rule) in (str, unicode): if rule in content: @@ -639,7 +646,7 @@ class Plugin(Base): if exists(location): raise SkipDownload(pyfile[0]) - self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) + self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name) def clean(self): -- cgit v1.2.3 From 9f2ebe486a3e155fb6a60e07cccb77ab6a772eb2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 26 Oct 2014 02:31:54 +0200 Subject: Extend translation support in plugins + a lot of code cosmetics and typo fixes --- module/plugins/Plugin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index efb20ad59..cc2fc794e 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -340,7 +340,7 @@ class Plugin(Base): :param reason: reason for retrying, will be passed to fail if max_tries reached """ if 0 < max_tries <= self.retries: - self.error(reason or "Max retries reached", "retry") + self.error(reason or _("Max retries reached"), "retry") self.wantReconnect = False self.setWait(wait_time) @@ -351,13 +351,13 @@ class Plugin(Base): def invalidCaptcha(self): - self.logError("Invalid captcha") + self.logError(_("Invalid captcha")) if self.cTask: self.cTask.invalid() def correctCaptcha(self): - self.logInfo("Correct captcha") + self.logInfo(_("Correct captcha")) if self.cTask: self.cTask.correct() @@ -419,7 +419,7 @@ class Plugin(Base): elif task.error: self.fail(task.error) elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) + self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) result = task.result self.logDebug("Received captcha result: %s" % str(result)) @@ -548,7 +548,7 @@ class Plugin(Base): self.pyfile.size = self.req.size if disposition and newname and newname != name: #triple check, just to be sure - self.logInfo("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) + self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) self.pyfile.name = newname filename = join(location, newname) @@ -639,7 +639,7 @@ class Plugin(Base): if starting and self.core.config['download']['skip_existing'] and exists(location): size = os.stat(location).st_size if size >= self.pyfile.size: - raise SkipDownload("File exists.") + raise SkipDownload("File exists") pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) if pyfile: -- cgit v1.2.3 From 4953542229356dce5ffca91fab88cad11ee8d07e Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 26 Oct 2014 19:34:15 +0100 Subject: Translation support for error method logging --- module/plugins/Plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index cc2fc794e..279edff4d 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -313,9 +313,9 @@ class Plugin(Base): if not reason and not type: type = "unknown" - msg = "%s error" % type.strip().capitalize() if type else "Error" + msg = _("%s error") % type.strip().capitalize() if type else _("Error") msg += ": " + reason.strip() if reason else "" - msg += " | Plugin may be out of date" + msg += _(" | Plugin may be out of date") if self.core.debug: print_exc() -- cgit v1.2.3 From 146fe1e309c33ab149bfaf58ad86c0dd4fb9b156 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Oct 2014 01:18:45 +0100 Subject: Spare code cosmetics --- module/plugins/Plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 279edff4d..18f51aab7 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -422,7 +422,7 @@ class Plugin(Base): self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) result = task.result - self.logDebug("Received captcha result: %s" % str(result)) + self.logDebug("Received captcha result: " + str(result)) if not self.core.debug: try: @@ -530,7 +530,7 @@ class Plugin(Base): chown(location, uid, gid) except Exception, e: - self.logWarning(_("Setting User and Group failed: %s") % str(e)) + self.logWarning(_("Setting User and Group failed: ") + str(e)) # convert back to unicode location = fs_decode(location) @@ -564,7 +564,7 @@ class Plugin(Base): chown(fs_filename, uid, gid) except Exception, e: - self.logWarning(_("Setting User and Group failed: %s") % str(e)) + self.logWarning(_("Setting User and Group failed: ") + str(e)) self.lastDownload = filename return self.lastDownload -- cgit v1.2.3 From b6f2bcc48d610e35df3038a9cdf2c07c39f62000 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 27 Oct 2014 23:03:29 +0100 Subject: [Plugin] Initial support for plugin deactivation --- module/plugins/Plugin.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 18f51aab7..8e5c01616 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -51,7 +51,6 @@ class Base(object): A Base class with log/config/db methods *all* plugin types can use """ - def __init__(self, core): #: Core instance self.core = core @@ -143,23 +142,25 @@ class Plugin(Base): Base plugin for hoster/crypter. Overwrite `process` / `decrypt` in your subclassed plugin. """ - __name__ = "Plugin" - __type__ = "hoster" + __name__ = "Plugin" + __type__ = "hoster" __version__ = "0.4" __pattern__ = None - __config__ = [("name", "type", "desc", "default")] + __config__ = [] #: [("name", "type", "desc", "default")] __description__ = """Base plugin""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("spoob", "spoob@pyload.org"), - ("mkaay", "mkaay@mkaay.de")] + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("spoob", "spoob@pyload.org"), + ("mkaay", "mkaay@mkaay.de")] def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) + self.__config__.insert(("activated", "bool", "Activated", True)) + self.wantReconnect = False #: enables simultaneous processing of multiple downloads self.multiDL = True @@ -313,7 +314,7 @@ class Plugin(Base): if not reason and not type: type = "unknown" - msg = _("%s error") % type.strip().capitalize() if type else _("Error") + msg = _("%s error") % _(type.strip().capitalize()) if type else _("Error") msg += ": " + reason.strip() if reason else "" msg += _(" | Plugin may be out of date") -- cgit v1.2.3 From 885f8ed782e64d9e73367905e642a84d0a8999f1 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 04:01:38 +0100 Subject: Update __config__ --- module/plugins/Plugin.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 8e5c01616..42bff9276 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -159,8 +159,6 @@ class Plugin(Base): def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) - self.__config__.insert(("activated", "bool", "Activated", True)) - self.wantReconnect = False #: enables simultaneous processing of multiple downloads self.multiDL = True -- cgit v1.2.3 From 9b08638a2b25b5ec38dc4091eefd73de0edfbe4f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 04:51:32 +0100 Subject: Improve debug logging in load/download method --- module/plugins/Plugin.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 42bff9276..11e1a06dd 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -449,7 +449,9 @@ class Plugin(Base): if type(url) == unicode: url = str(url) # encode('utf8') - self.logDebug("Load url", *["%s: %s" % (key, val) for key, val in locals().items()]) + if self.core.debug: + kwargs = locals() + self.logDebug("Load url: " + kwargs['url'], *["%s=%s" % (key, val) for key, val in kwargs.iteritems() if key not in ("self", "url")]) res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) @@ -509,7 +511,9 @@ class Plugin(Base): :return: The location where the file was saved """ - self.logDebug("Download url", *["%s: %s" % (key, val) for key, val in locals().items()]) + if self.core.debug: + kwargs = locals() + self.logDebug("Download url: " + kwargs['url'], *["%s=%s" % (key, val) for key, val in kwargs.iteritems() if key not in ("self", "url")]) self.checkForSameFiles() -- cgit v1.2.3 From 9f9cfaa556071830cbbcc6a740dd6df0ad7ca393 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 28 Oct 2014 16:00:48 +0100 Subject: [Plugin] Improve retry and wait methods --- module/plugins/Plugin.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 11e1a06dd..be2c14811 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -274,19 +274,23 @@ class Plugin(Base): return True, 10 - def setWait(self, seconds, reconnect=False): + def setWait(self, seconds, reconnect=None): """Set a specific wait time later used with `wait` :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ - if reconnect: - self.wantReconnect = True - self.pyfile.waitUntil = time() + int(seconds) + if reconnect is not None: + self.wantReconnect = bool(reconnect) + self.pyfile.waitUntil = time() + int(seconds) + 1 - def wait(self): + def wait(self, seconds=0, reconnect=None): """ waits the time previously set """ + + if seconds: + self.setWait(seconds, reconnect) + self.waiting = True self.pyfile.setStatus("waiting") @@ -331,7 +335,7 @@ class Plugin(Base): raise Fail("temp. offline") - def retry(self, max_tries=3, wait_time=1, reason=""): + def retry(self, max_tries=5, wait_time=1, reason=""): """Retries and begin again from the beginning :param max_tries: number of maximum retries @@ -341,9 +345,7 @@ class Plugin(Base): if 0 < max_tries <= self.retries: self.error(reason or _("Max retries reached"), "retry") - self.wantReconnect = False - self.setWait(wait_time) - self.wait() + self.wait(wait_time, False) self.retries += 1 raise Retry(reason) -- cgit v1.2.3 From 2c7d777216d36c13403867d37f4a584676990502 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 30 Oct 2014 17:21:54 +0100 Subject: [Plugin] Improve debug logging of load and download methods + url striping --- module/plugins/Plugin.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index be2c14811..241c2c12a 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -180,7 +180,9 @@ class Plugin(Base): #: username/login self.user = None - if self.account and not self.account.canUse(): self.account = None + if self.account and not self.account.canUse(): + self.account = None + if self.account: self.user, data = self.account.selectAccount() #: Browser instance, see `network.Browser` @@ -446,14 +448,16 @@ class Plugin(Base): :param decode: Wether to decode the output according to http header, should be True in most cases :return: Loaded content """ - if self.pyfile.abort: raise Abort + if self.pyfile.abort: + raise Abort + + url = url.strip() #utf8 vs decode -> please use decode attribute in all future plugins if type(url) == unicode: url = str(url) # encode('utf8') if self.core.debug: - kwargs = locals() - self.logDebug("Load url: " + kwargs['url'], *["%s=%s" % (key, val) for key, val in kwargs.iteritems() if key not in ("self", "url")]) + self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) @@ -512,10 +516,13 @@ class Plugin(Base): the filename will be changed if needed :return: The location where the file was saved """ + if self.pyfile.abort: + raise Abort + + url = url.strip() if self.core.debug: - kwargs = locals() - self.logDebug("Download url: " + kwargs['url'], *["%s=%s" % (key, val) for key, val in kwargs.iteritems() if key not in ("self", "url")]) + self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) self.checkForSameFiles() -- cgit v1.2.3 From 34588ba93bd9a2dad164cca7e9702083d3d373ad Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 1 Nov 2014 18:39:07 +0100 Subject: [Plugin] Declare self.html as empty string, not as None type --- module/plugins/Plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 241c2c12a..7db733dc2 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -209,7 +209,7 @@ class Plugin(Base): self.cTask = None #captcha task self.retries = 0 # amount of retries already made - self.html = None # some plugins store html code here + self.html = "" # some plugins store html code here self.init() -- cgit v1.2.3 From 75379fd60a82ff78ba5cc9368088ccb37d318d22 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 2 Nov 2014 22:31:17 +0100 Subject: [Plugin] Improve logging --- module/plugins/Plugin.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 7db733dc2..feb81b096 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -60,25 +60,30 @@ class Base(object): self.config = core.config - #log functions + def _log(self, type, args): + msg = " | ".join([str(a).strip() for a in args if a]) + logger = getattr(self.log, type) + logger("%s: %s" % (self.__name__, msg or _("%s MARK" % type.upper()))) + + def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([str(a).strip() for a in args if a]))) + return self._log("debug", args) def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([str(a).strip() for a in args if a]))) + return self._log("info", args) def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([str(a).strip() for a in args if a]))) + return self._log("warning", args) def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([str(a).strip() for a in args if a]))) + return self._log("error", args) def logCritical(self, *args): - self.log.critical("%s: %s" % (self.__name__, " | ".join([str(a).strip() for a in args if a]))) + return self._log("critical", args) def setConf(self, option, value): @@ -209,7 +214,7 @@ class Plugin(Base): self.cTask = None #captcha task self.retries = 0 # amount of retries already made - self.html = "" # some plugins store html code here + self.html = None # some plugins store html code here self.init() -- cgit v1.2.3 From cb5e1c3439a27d28df496b582bb38e1e9ed649cb Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Nov 2014 12:10:11 +0100 Subject: [Plugin] Increase default captcha timeout to 5 minutes from 50 seconds --- module/plugins/Plugin.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index feb81b096..8855e5980 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -149,7 +149,7 @@ class Plugin(Base): """ __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.4" + __version__ = "0.05" __pattern__ = None __config__ = [] #: [("name", "type", "desc", "default")] @@ -214,7 +214,6 @@ class Plugin(Base): self.cTask = None #captcha task self.retries = 0 # amount of retries already made - self.html = None # some plugins store html code here self.init() @@ -371,7 +370,7 @@ class Plugin(Base): def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual'): + result_type='textual', timeout=290): """ Loads a captcha and decrypts it with ocr, plugin, user input :param url: url of captcha image @@ -412,7 +411,7 @@ class Plugin(Base): captchaManager = self.core.captchaManager task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) self.cTask = task - captchaManager.handleCaptcha(task) + captchaManager.handleCaptcha(task, timeout) while task.isWaiting(): if self.pyfile.abort: @@ -675,5 +674,5 @@ class Plugin(Base): del self.req if hasattr(self, "thread"): del self.thread - if hasattr(self, "html"): - del self.html + # if hasattr(self, "html"): + # del self.html -- cgit v1.2.3 From 03f3b86f500c495932fd118b54569d92f700847c Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 3 Nov 2014 16:57:55 +0100 Subject: Code cosmetics about file_info and other stuff --- module/plugins/Plugin.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 8855e5980..91bf3aab9 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -161,6 +161,9 @@ class Plugin(Base): ("mkaay", "mkaay@mkaay.de")] + info = {} #: file info dict + + def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) -- cgit v1.2.3 From 2f4d7d614171054c3c2b0b6185fa4ea465f5312f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Tue, 4 Nov 2014 01:09:43 +0100 Subject: Use more print_exc + code cosmetics --- module/plugins/Plugin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 91bf3aab9..bf051a33f 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -306,7 +306,8 @@ class Plugin(Base): while self.pyfile.waitUntil > time(): self.thread.m.reconnecting.wait(2) - if self.pyfile.abort: raise Abort + if self.pyfile.abort: + raise Abort if self.thread.m.reconnecting.isSet(): self.waiting = False self.wantReconnect = False @@ -329,8 +330,6 @@ class Plugin(Base): msg += ": " + reason.strip() if reason else "" msg += _(" | Plugin may be out of date") - if self.core.debug: - print_exc() raise Fail(msg) -- cgit v1.2.3 From e1981f7e97ee79b08205a9f9a7367867b847d49a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 6 Nov 2014 03:31:58 +0100 Subject: [Plugin] Don't use reconnection if account was logged --- module/plugins/Plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index bf051a33f..a74a95e78 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -127,7 +127,7 @@ class Base(object): def getStorage(self, key=None, default=None): """ Retrieves saved value or dict of all saved entries if key is None """ - if key is not None: + if key: return self.core.db.getStorage(self.__name__, key) or default return self.core.db.getStorage(self.__name__, key) @@ -289,7 +289,7 @@ class Plugin(Base): :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ - if reconnect is not None: + if reconnect and not self.account: self.wantReconnect = bool(reconnect) self.pyfile.waitUntil = time() + int(seconds) + 1 @@ -308,6 +308,7 @@ class Plugin(Base): if self.pyfile.abort: raise Abort + if self.thread.m.reconnecting.isSet(): self.waiting = False self.wantReconnect = False -- cgit v1.2.3 From 721783bae67cd78ce8c171f8e87d5b4ea6836c51 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 6 Nov 2014 03:49:31 +0100 Subject: [Plugin] Don't use reconnection if account was logged 2 --- module/plugins/Plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index a74a95e78..cdf0f147d 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -289,8 +289,8 @@ class Plugin(Base): :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ - if reconnect and not self.account: - self.wantReconnect = bool(reconnect) + if reconnect is not None: + self.wantReconnect = reconnect and not self.account self.pyfile.waitUntil = time() + int(seconds) + 1 -- cgit v1.2.3 From 6bcfdef32e7e62f8f749ceb6618ee7550bce28de Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 6 Nov 2014 22:42:59 +0100 Subject: Lowercase url to load --- module/plugins/Plugin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index cdf0f147d..f80eb461b 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -458,10 +458,10 @@ class Plugin(Base): if self.pyfile.abort: raise Abort - url = url.strip() - #utf8 vs decode -> please use decode attribute in all future plugins - if type(url) == unicode: - url = str(url) # encode('utf8') + url = url.strip().lower() + + if type(url) == unicode: # utf8 vs decode -> please use decode attribute in all future plugins + url = str(url) #: encode('utf8') if self.core.debug: self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) @@ -496,7 +496,7 @@ class Plugin(Base): if not line or ":" not in line: continue key, none, value = line.partition(":") - key = key.lower().strip() + key = key.strip().lower() value = value.strip() if key in header: @@ -526,7 +526,7 @@ class Plugin(Base): if self.pyfile.abort: raise Abort - url = url.strip() + url = url.strip().lower() if self.core.debug: self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) -- cgit v1.2.3 From 58c9c8d78417ec6ccac5b6b117ea540033a09574 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 7 Nov 2014 17:08:21 +0100 Subject: Fix download link formatting in some plugins --- module/plugins/Plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index f80eb461b..426a406a9 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -458,7 +458,7 @@ class Plugin(Base): if self.pyfile.abort: raise Abort - url = url.strip().lower() + url = url.strip() if type(url) == unicode: # utf8 vs decode -> please use decode attribute in all future plugins url = str(url) #: encode('utf8') @@ -526,7 +526,7 @@ class Plugin(Base): if self.pyfile.abort: raise Abort - url = url.strip().lower() + url = url.strip() if self.core.debug: self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) -- cgit v1.2.3 From 344f22f9ba8b3aa4e7d8049084db473b76cfb4e3 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 9 Nov 2014 21:48:16 +0100 Subject: [Plugin] Update and bugfix --- module/plugins/Plugin.py | 133 +++++++++++++++++++++++++++++++---------------- 1 file changed, 88 insertions(+), 45 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 426a406a9..a8eca5003 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -14,8 +14,9 @@ if os.name != "nt": from itertools import islice from traceback import print_exc +from urlparse import urlparse -from module.utils import save_join, save_path, fs_encode, fs_decode +from module.utils import fs_decode, fs_encode, html_unescape, save_join, save_path def chunks(iterable, size): @@ -67,7 +68,8 @@ class Base(object): def logDebug(self, *args): - return self._log("debug", args) + if self.core.debug: + return self._log("debug", args) def logInfo(self, *args): @@ -149,7 +151,7 @@ class Plugin(Base): """ __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.05" + __version__ = "0.06" __pattern__ = None __config__ = [] #: [("name", "type", "desc", "default")] @@ -216,6 +218,7 @@ class Plugin(Base): self.js = self.core.js self.cTask = None #captcha task + self.html = None #@TODO: Move to hoster class self.retries = 0 # amount of retries already made self.init() @@ -289,33 +292,56 @@ class Plugin(Base): :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ + wait_time = int(seconds) + 1 + wait_until = time() + wait_time + + self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.self.pyfile.waitUntil), + "Wait: %d seconds" % wait_time) + + self.pyfile.waitUntil = wait_until + if reconnect is not None: - self.wantReconnect = reconnect and not self.account - self.pyfile.waitUntil = time() + int(seconds) + 1 + self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) + self.wantReconnect = reconnect def wait(self, seconds=0, reconnect=None): """ waits the time previously set """ - if seconds: - self.setWait(seconds, reconnect) + pyfile = self.pyfile + + self.setWait(seconds, reconnect) self.waiting = True - self.pyfile.setStatus("waiting") - while self.pyfile.waitUntil > time(): - self.thread.m.reconnecting.wait(2) + status = pyfile.status + pyfile.setStatus("waiting") - if self.pyfile.abort: - raise Abort + self.logDebug("WAIT: %d seconds" % wait_time, + "WAITUNTIL: %f" % pyfile.waitUntil, + "RECONNECT: %s" % self.wantReconnect) + + if not account: + self.logDebug("Ignore reconnection due account logged") - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect + while pyfile.waitUntil > time(): + self.thread.m.reconnecting.wait(2) + + if pyfile.abort: + self.abort() + + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + else: + while pyfile.waitUntil > time(): + if pyfile.abort: + self.abort() self.waiting = False - self.pyfile.setStatus("starting") + + pyfile.status = status def fail(self, reason): @@ -323,6 +349,11 @@ class Plugin(Base): raise Fail(reason) + def abort(self, reason=""): + """ abort and give reason """ + raise Abort + + def error(self, reason="", type=""): if not reason and not type: type = "unknown" @@ -392,9 +423,9 @@ class Plugin(Base): img = self.load(url, get=get, post=post, cookies=cookies) id = ("%.2f" % time())[-6:].replace(".", "") - tmpCaptcha = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - tmpCaptcha.write(img) - tmpCaptcha.close() + + with open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha: + tmpCaptcha.write(img) has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins @@ -406,7 +437,7 @@ class Plugin(Base): if Ocr and not forceUser: sleep(randint(3000, 5000) / 1000.0) if self.pyfile.abort: - raise Abort + self.abort() ocr = Ocr() result = ocr.get_captcha(tmpCaptcha.name) @@ -419,7 +450,7 @@ class Plugin(Base): while task.isWaiting(): if self.pyfile.abort: captchaManager.removeTask(task) - raise Abort + self.abort() sleep(1) captchaManager.removeTask(task) @@ -443,7 +474,7 @@ class Plugin(Base): return result - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True): """Load content at url and returns it :param url: @@ -451,22 +482,27 @@ class Plugin(Base): :param post: :param ref: :param cookies: - :param just_header: if True only the header will be retrieved and returned as dict + :param just_header: If True only the header will be retrieved and returned as dict :param decode: Wether to decode the output according to http header, should be True in most cases + :param follow_location: If True follow location else not + :param save_cookies: If True saves received cookies else discard them :return: Loaded content """ if self.pyfile.abort: - raise Abort + self.abort() - url = url.strip() + if not url: + self.fail(_"No url given")) if type(url) == unicode: # utf8 vs decode -> please use decode attribute in all future plugins url = str(url) #: encode('utf8') + url = url.strip() + if self.core.debug: self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) - res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) + res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies) if self.core.debug: from inspect import currentframe @@ -475,18 +511,13 @@ class Plugin(Base): if not exists(join("tmp", self.__name__)): makedirs(join("tmp", self.__name__)) - f = open( - join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) - , "wb") - del frame # delete the frame or it wont be cleaned - + framefile = save_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) try: - tmp = res.encode("utf8") - except: - tmp = res - - f.write(tmp) - f.close() + with open(framefile, "wb") as f: + del frame #: delete the frame or it wont be cleaned + f.write(fs_encode(res)) + except IOError, e: + self.logError(str(e)) if just_header: #parse header @@ -524,7 +555,13 @@ class Plugin(Base): :return: The location where the file was saved """ if self.pyfile.abort: - raise Abort + self.abort() + + if not url: + self.fail(_"No url given")) + + if type(url) == unicode: + url = str(url) url = url.strip() @@ -563,6 +600,7 @@ class Plugin(Base): newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, chunks=self.getChunkCount(), resume=self.resumeDownload, progressNotify=self.pyfile.setProgress, disposition=disposition) + newname = urlparse(html_unescape(newname)).path.split("/")[-1] finally: self.pyfile.size = self.req.size @@ -600,7 +638,8 @@ class Plugin(Base): :return: dictionary key of the first rule that matched """ lastDownload = fs_encode(self.lastDownload) - if not exists(lastDownload): return None + if not exists(lastDownload): + return None size = stat(lastDownload) size = size.st_size @@ -608,9 +647,10 @@ class Plugin(Base): if api_size and api_size <= size: return None elif size > max_size and not read_size: return None self.logDebug("Download Check triggered") - f = open(lastDownload, "rb") - content = f.read(read_size if read_size else -1) - f.close() + + with open(lastDownload, "rb") as f: + content = f.read(read_size if read_size else -1) + #produces encoding errors, better log to other file in the future? #self.logDebug("Content: %s" % content) for name, rule in rules.iteritems(): @@ -672,10 +712,13 @@ class Plugin(Base): """ clean everything and remove references """ if hasattr(self, "pyfile"): del self.pyfile + if hasattr(self, "req"): self.req.close() del self.req + if hasattr(self, "thread"): del self.thread - # if hasattr(self, "html"): - # del self.html + + if hasattr(self, "html"): + del self.html -- cgit v1.2.3 From c9e31d875d32de31e54959b82bc35eff2b3e0f3f Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 10 Nov 2014 00:19:51 +0100 Subject: Code cosmetics --- module/plugins/Plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index a8eca5003..f296533e1 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -517,7 +517,7 @@ class Plugin(Base): del frame #: delete the frame or it wont be cleaned f.write(fs_encode(res)) except IOError, e: - self.logError(str(e)) + self.logError(e) if just_header: #parse header -- cgit v1.2.3 From 394a9b9f0404014f42090495c87520d23aff204a Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 10 Nov 2014 01:27:47 +0100 Subject: [Plugin] Fix typo (thx clinton-hall) --- module/plugins/Plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index f296533e1..d21bbb196 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -151,7 +151,7 @@ class Plugin(Base): """ __name__ = "Plugin" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.07" __pattern__ = None __config__ = [] #: [("name", "type", "desc", "default")] @@ -492,7 +492,7 @@ class Plugin(Base): self.abort() if not url: - self.fail(_"No url given")) + self.fail(_("No url given")) if type(url) == unicode: # utf8 vs decode -> please use decode attribute in all future plugins url = str(url) #: encode('utf8') @@ -558,7 +558,7 @@ class Plugin(Base): self.abort() if not url: - self.fail(_"No url given")) + self.fail(_("No url given")) if type(url) == unicode: url = str(url) -- cgit v1.2.3 From ff0e1a28ce6ac27d11cea7cceaeb50a779facafb Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 12 Nov 2014 01:52:08 +0100 Subject: [Plugin] Save reason as pyfile.error when fails --- module/plugins/Plugin.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index d21bbb196..d1227eb29 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -351,7 +351,9 @@ class Plugin(Base): def abort(self, reason=""): """ abort and give reason """ - raise Abort + if reason: + self.pyfile.error = str(reason) + raise Abort #@TODO: Use raise Abort(reason) in 0.4.10 def error(self, reason="", type=""): @@ -365,14 +367,18 @@ class Plugin(Base): raise Fail(msg) - def offline(self): + def offline(self, reason=""): """ fail and indicate file is offline """ - raise Fail("offline") + if reason: + self.pyfile.error = str(reason) + raise Fail("offline") #@TODO: Use raise Fail("offline", reason) in 0.4.10 - def tempOffline(self): + def tempOffline(self, reason=""): """ fail and indicates file ist temporary offline, the core may take consequences """ - raise Fail("temp. offline") + if reason: + self.pyfile.error = str(reason) + raise Fail("temp. offline") #@TODO: Use raise Fail("temp. offline", reason) in 0.4.10 def retry(self, max_tries=5, wait_time=1, reason=""): -- cgit v1.2.3 From 41326b63c836a91aff560181b7bcc2099b0eca88 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 12 Nov 2014 16:11:39 +0100 Subject: [Plugin] Content-disposition name should not need to be html_unescaped --- module/plugins/Plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index d1227eb29..f4b6c85cd 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -606,7 +606,7 @@ class Plugin(Base): newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, chunks=self.getChunkCount(), resume=self.resumeDownload, progressNotify=self.pyfile.setProgress, disposition=disposition) - newname = urlparse(html_unescape(newname)).path.split("/")[-1] + newname = urlparse(newname).path.split("/")[-1] finally: self.pyfile.size = self.req.size -- cgit v1.2.3 From f724290003013c6167918c220bf76a1c5cd58543 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Wed, 12 Nov 2014 22:14:42 +0100 Subject: [Plugin] Fix download routine --- module/plugins/Plugin.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index f4b6c85cd..7bd29c9d6 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -514,11 +514,11 @@ class Plugin(Base): from inspect import currentframe frame = currentframe() - if not exists(join("tmp", self.__name__)): - makedirs(join("tmp", self.__name__)) - framefile = save_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) try: + if not exists(join("tmp", self.__name__)): + makedirs(join("tmp", self.__name__)) + with open(framefile, "wb") as f: del frame #: delete the frame or it wont be cleaned f.write(fs_encode(res)) @@ -583,16 +583,16 @@ class Plugin(Base): location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): - makedirs(location, int(self.core.config['permission']['folder'], 8)) + try: + makedirs(location, int(self.core.config['permission']['folder'], 8)) - if self.core.config['permission']['change_dl'] and os.name != "nt": - try: + if self.core.config['permission']['change_dl'] and os.name != "nt": uid = getpwnam(self.config['permission']['user'])[2] gid = getgrnam(self.config['permission']['group'])[2] - chown(location, uid, gid) - except Exception, e: - self.logWarning(_("Setting User and Group failed: ") + str(e)) + + except Exception, e: + self.fail(e) # convert back to unicode location = fs_decode(location) @@ -606,28 +606,33 @@ class Plugin(Base): newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, chunks=self.getChunkCount(), resume=self.resumeDownload, progressNotify=self.pyfile.setProgress, disposition=disposition) - newname = urlparse(newname).path.split("/")[-1] finally: self.pyfile.size = self.req.size - if disposition and newname and newname != name: #triple check, just to be sure - self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) - self.pyfile.name = newname - filename = join(location, newname) + if newname: + newname = urlparse(newname).path.split("/")[-1] + + if disposition and newname != name: + self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = join(location, newname) fs_filename = fs_encode(filename) if self.core.config['permission']['change_file']: - chmod(fs_filename, int(self.core.config['permission']['file'], 8)) + try: + chmod(fs_filename, int(self.core.config['permission']['file'], 8)) + except Exception, e: + self.logWarning(_("Setting file mode failed"), e) if self.core.config['permission']['change_dl'] and os.name != "nt": try: uid = getpwnam(self.config['permission']['user'])[2] gid = getgrnam(self.config['permission']['group'])[2] - chown(fs_filename, uid, gid) + except Exception, e: - self.logWarning(_("Setting User and Group failed: ") + str(e)) + self.logWarning(_("Setting User and Group failed"), e) self.lastDownload = filename return self.lastDownload -- cgit v1.2.3 From 21a2805840d283f23e8dc0fded2fdc450de1fd8b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Thu, 13 Nov 2014 10:49:58 +0100 Subject: [Plugin] Some fixes by rlindner81 --- module/plugins/Plugin.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 7bd29c9d6..120dc9a6e 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -295,7 +295,7 @@ class Plugin(Base): wait_time = int(seconds) + 1 wait_until = time() + wait_time - self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.self.pyfile.waitUntil), + self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), "Wait: %d seconds" % wait_time) self.pyfile.waitUntil = wait_until @@ -317,9 +317,9 @@ class Plugin(Base): status = pyfile.status pyfile.setStatus("waiting") - self.logDebug("WAIT: %d seconds" % wait_time, - "WAITUNTIL: %f" % pyfile.waitUntil, - "RECONNECT: %s" % self.wantReconnect) + self.logDebug("WAIT: %d seconds" % seconds, + "WAITUNTIL: %f" % pyfile.waitUntil, + "RECONNECT: %s" % self.wantReconnect) if not account: self.logDebug("Ignore reconnection due account logged") @@ -521,7 +521,9 @@ class Plugin(Base): with open(framefile, "wb") as f: del frame #: delete the frame or it wont be cleaned - f.write(fs_encode(res)) + if decode: + res = res.encode('utf-8') + f.write(res) except IOError, e: self.logError(e) -- cgit v1.2.3 From a0fc1dd7dba89dccdb730f8ea0259e4ac9ba7e08 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 14 Nov 2014 00:12:31 +0100 Subject: [Plugin] Fix wait routine (thx rlindner81) --- module/plugins/Plugin.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 120dc9a6e..4dadb67b4 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -286,6 +286,12 @@ class Plugin(Base): return True, 10 + def setReconnect(self, reconnect): + reconnect = bool(reconnect) + self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) + self.wantReconnect = reconnect + + def setWait(self, seconds, reconnect=None): """Set a specific wait time later used with `wait` @@ -301,16 +307,19 @@ class Plugin(Base): self.pyfile.waitUntil = wait_until if reconnect is not None: - self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) - self.wantReconnect = reconnect + self.setReconnect(reconnect) - def wait(self, seconds=0, reconnect=None): + def wait(self, seconds=None, reconnect=None): """ waits the time previously set """ pyfile = self.pyfile - self.setWait(seconds, reconnect) + if seconds is not None: + self.setWait(seconds) + + if reconnect is not None: + self.setReconnect(reconnect) self.waiting = True @@ -321,9 +330,13 @@ class Plugin(Base): "WAITUNTIL: %f" % pyfile.waitUntil, "RECONNECT: %s" % self.wantReconnect) - if not account: + if self.account: self.logDebug("Ignore reconnection due account logged") + while pyfile.waitUntil > time(): + if pyfile.abort: + self.abort() + else: while pyfile.waitUntil > time(): self.thread.m.reconnecting.wait(2) @@ -334,10 +347,6 @@ class Plugin(Base): self.waiting = False self.wantReconnect = False raise Reconnect - else: - while pyfile.waitUntil > time(): - if pyfile.abort: - self.abort() self.waiting = False -- cgit v1.2.3 From 38a5cae36ebeb1894fa3c6a5c2f2eb2d58fe293b Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 15 Nov 2014 01:39:26 +0100 Subject: [Plugin] Fix wait routine 2 --- module/plugins/Plugin.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 4dadb67b4..1be74e896 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -326,9 +326,8 @@ class Plugin(Base): status = pyfile.status pyfile.setStatus("waiting") - self.logDebug("WAIT: %d seconds" % seconds, - "WAITUNTIL: %f" % pyfile.waitUntil, - "RECONNECT: %s" % self.wantReconnect) + self.logInfo(_("Wait: %d seconds") % pyfile.waitUntil - time(), + _("Reconnect: %s") % self.wantReconnect) if self.account: self.logDebug("Ignore reconnection due account logged") @@ -336,6 +335,8 @@ class Plugin(Base): while pyfile.waitUntil > time(): if pyfile.abort: self.abort() + + sleep(1) else: while pyfile.waitUntil > time(): self.thread.m.reconnecting.wait(2) @@ -348,6 +349,8 @@ class Plugin(Base): self.wantReconnect = False raise Reconnect + sleep(1) + self.waiting = False pyfile.status = status -- cgit v1.2.3 From 253ec9382068a2cc255f6a1e93005ee3da7965a2 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 15 Nov 2014 17:21:10 +0100 Subject: [Plugin] Fix typo (thx rlindner81) --- module/plugins/Plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 1be74e896..55d93e357 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -326,7 +326,7 @@ class Plugin(Base): status = pyfile.status pyfile.setStatus("waiting") - self.logInfo(_("Wait: %d seconds") % pyfile.waitUntil - time(), + self.logInfo(_("Wait: %d seconds") % (pyfile.waitUntil - time()), _("Reconnect: %s") % self.wantReconnect) if self.account: -- cgit v1.2.3 From e08fb5d3361627ad866aed86be9597b32c8c50f9 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Fri, 21 Nov 2014 18:23:33 +0100 Subject: [SimpleHoster] Code cosmetics --- module/plugins/Plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 55d93e357..550169034 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -16,7 +16,7 @@ from itertools import islice from traceback import print_exc from urlparse import urlparse -from module.utils import fs_decode, fs_encode, html_unescape, save_join, save_path +from module.utils import fs_decode, fs_encode, save_join, save_path def chunks(iterable, size): -- cgit v1.2.3 From ef4bc4b73756565e40c7453f6b71bc1021735033 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sat, 22 Nov 2014 19:38:25 +0100 Subject: Revert plugins to stable --- module/plugins/Plugin.py | 376 +++++++++++++++-------------------------------- 1 file changed, 122 insertions(+), 254 deletions(-) (limited to 'module/plugins/Plugin.py') diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 550169034..15bf3971f 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -1,5 +1,22 @@ # -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN, spoob, mkaay +""" + from time import time, sleep from random import randint @@ -13,11 +30,8 @@ if os.name != "nt": from grp import getgrnam from itertools import islice -from traceback import print_exc -from urlparse import urlparse - -from module.utils import fs_decode, fs_encode, save_join, save_path +from module.utils import save_join, save_path, fs_encode, fs_decode def chunks(iterable, size): it = iter(iterable) @@ -60,39 +74,24 @@ class Base(object): #: core config self.config = core.config - - def _log(self, type, args): - msg = " | ".join([str(a).strip() for a in args if a]) - logger = getattr(self.log, type) - logger("%s: %s" % (self.__name__, msg or _("%s MARK" % type.upper()))) - - - def logDebug(self, *args): - if self.core.debug: - return self._log("debug", args) - - + #log functions def logInfo(self, *args): - return self._log("info", args) - + self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) def logWarning(self, *args): - return self._log("warning", args) - + self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) def logError(self, *args): - return self._log("error", args) + self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logCritical(self, *args): - return self._log("critical", args) + def logDebug(self, *args): + self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) def setConf(self, option, value): """ see `setConfig` """ self.core.config.setPlugin(self.__name__, option, value) - def setConfig(self, option, value): """ Set config value for current plugin @@ -102,12 +101,10 @@ class Base(object): """ self.setConf(option, value) - def getConf(self, option): """ see `getConfig` """ return self.core.config.getPlugin(self.__name__, option) - def getConfig(self, option): """ Returns config value for current plugin @@ -116,29 +113,24 @@ class Base(object): """ return self.getConf(option) - def setStorage(self, key, value): """ Saves a value persistently to the database """ self.core.db.setStorage(self.__name__, key, value) - def store(self, key, value): """ same as `setStorage` """ self.core.db.setStorage(self.__name__, key, value) - def getStorage(self, key=None, default=None): """ Retrieves saved value or dict of all saved entries if key is None """ - if key: + if key is not None: return self.core.db.getStorage(self.__name__, key) or default return self.core.db.getStorage(self.__name__, key) - def retrieve(self, *args, **kwargs): """ same as `getStorage` """ return self.getStorage(*args, **kwargs) - def delStorage(self, key): """ Delete entry in db """ self.core.db.delStorage(self.__name__, key) @@ -149,22 +141,14 @@ class Plugin(Base): Base plugin for hoster/crypter. Overwrite `process` / `decrypt` in your subclassed plugin. """ - __name__ = "Plugin" - __type__ = "hoster" - __version__ = "0.07" - + __name__ = "Plugin" + __version__ = "0.4" __pattern__ = None - __config__ = [] #: [("name", "type", "desc", "default")] - - __description__ = """Base plugin""" - __license__ = "GPLv3" - __authors__ = [("RaNaN", "RaNaN@pyload.org"), - ("spoob", "spoob@pyload.org"), - ("mkaay", "mkaay@mkaay.de")] - - - info = {} #: file info dict - + __type__ = "hoster" + __config__ = [("name", "type", "desc", "default")] + __description__ = """Base Plugin""" + __author_name__ = ("RaNaN", "spoob", "mkaay") + __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) @@ -190,9 +174,7 @@ class Plugin(Base): #: username/login self.user = None - if self.account and not self.account.canUse(): - self.account = None - + if self.account and not self.account.canUse(): self.account = None if self.account: self.user, data = self.account.selectAccount() #: Browser instance, see `network.Browser` @@ -218,32 +200,27 @@ class Plugin(Base): self.js = self.core.js self.cTask = None #captcha task - self.html = None #@TODO: Move to hoster class self.retries = 0 # amount of retries already made + self.html = None # some plugins store html code here self.init() - def getChunkCount(self): if self.chunkLimit <= 0: - return self.config['download']['chunks'] - return min(self.config['download']['chunks'], self.chunkLimit) - + return self.config["download"]["chunks"] + return min(self.config["download"]["chunks"], self.chunkLimit) def __call__(self): return self.__name__ - def init(self): """initialize the plugin (in addition to `__init__`)""" pass - def setup(self): """ setup for enviroment and other things, called before downloading (possibly more than one time)""" pass - def preprocessing(self, thread): """ handles important things to do before starting """ self.thread = thread @@ -264,14 +241,12 @@ class Plugin(Base): """the 'main' method of every plugin, you **have to** overwrite it""" raise NotImplementedError - def resetAccount(self): """ dont use account and retry download """ self.account = None self.req = self.core.requestFactory.getRequest(self.__name__) self.retry() - def checksum(self, local_file=None): """ return codes: @@ -286,114 +261,46 @@ class Plugin(Base): return True, 10 - def setReconnect(self, reconnect): - reconnect = bool(reconnect) - self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect)) - self.wantReconnect = reconnect - - - def setWait(self, seconds, reconnect=None): + def setWait(self, seconds, reconnect=False): """Set a specific wait time later used with `wait` - + :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ - wait_time = int(seconds) + 1 - wait_until = time() + wait_time - - self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil), - "Wait: %d seconds" % wait_time) + if reconnect: + self.wantReconnect = True + self.pyfile.waitUntil = time() + int(seconds) - self.pyfile.waitUntil = wait_until - - if reconnect is not None: - self.setReconnect(reconnect) - - - def wait(self, seconds=None, reconnect=None): + def wait(self): """ waits the time previously set """ - - pyfile = self.pyfile - - if seconds is not None: - self.setWait(seconds) - - if reconnect is not None: - self.setReconnect(reconnect) - self.waiting = True + self.pyfile.setStatus("waiting") - status = pyfile.status - pyfile.setStatus("waiting") + while self.pyfile.waitUntil > time(): + self.thread.m.reconnecting.wait(2) - self.logInfo(_("Wait: %d seconds") % (pyfile.waitUntil - time()), - _("Reconnect: %s") % self.wantReconnect) - - if self.account: - self.logDebug("Ignore reconnection due account logged") - - while pyfile.waitUntil > time(): - if pyfile.abort: - self.abort() - - sleep(1) - else: - while pyfile.waitUntil > time(): - self.thread.m.reconnecting.wait(2) - - if pyfile.abort: - self.abort() - - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect - - sleep(1) + if self.pyfile.abort: raise Abort + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect self.waiting = False - - pyfile.status = status - + self.pyfile.setStatus("starting") def fail(self, reason): """ fail and give reason """ raise Fail(reason) - - def abort(self, reason=""): - """ abort and give reason """ - if reason: - self.pyfile.error = str(reason) - raise Abort #@TODO: Use raise Abort(reason) in 0.4.10 - - - def error(self, reason="", type=""): - if not reason and not type: - type = "unknown" - - msg = _("%s error") % _(type.strip().capitalize()) if type else _("Error") - msg += ": " + reason.strip() if reason else "" - msg += _(" | Plugin may be out of date") - - raise Fail(msg) - - - def offline(self, reason=""): + def offline(self): """ fail and indicate file is offline """ - if reason: - self.pyfile.error = str(reason) - raise Fail("offline") #@TODO: Use raise Fail("offline", reason) in 0.4.10 + raise Fail("offline") - - def tempOffline(self, reason=""): + def tempOffline(self): """ fail and indicates file ist temporary offline, the core may take consequences """ - if reason: - self.pyfile.error = str(reason) - raise Fail("temp. offline") #@TODO: Use raise Fail("temp. offline", reason) in 0.4.10 - + raise Fail("temp. offline") - def retry(self, max_tries=5, wait_time=1, reason=""): + def retry(self, max_tries=3, wait_time=1, reason=""): """Retries and begin again from the beginning :param max_tries: number of maximum retries @@ -401,28 +308,26 @@ class Plugin(Base): :param reason: reason for retrying, will be passed to fail if max_tries reached """ if 0 < max_tries <= self.retries: - self.error(reason or _("Max retries reached"), "retry") + if not reason: reason = "Max retries reached" + raise Fail(reason) - self.wait(wait_time, False) + self.wantReconnect = False + self.setWait(wait_time) + self.wait() self.retries += 1 raise Retry(reason) - def invalidCaptcha(self): - self.logError(_("Invalid captcha")) if self.cTask: self.cTask.invalid() - def correctCaptcha(self): - self.logInfo(_("Correct captcha")) if self.cTask: self.cTask.correct() - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual', timeout=290): + result_type='textual'): """ Loads a captcha and decrypts it with ocr, plugin, user input :param url: url of captcha image @@ -434,16 +339,16 @@ class Plugin(Base): :param result_type: 'textual' if text is written on the captcha\ or 'positional' for captcha where the user have to click\ on a specific region on the captcha - + :return: result of decrypting """ img = self.load(url, get=get, post=post, cookies=cookies) id = ("%.2f" % time())[-6:].replace(".", "") - - with open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha: - tmpCaptcha.write(img) + temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + temp_file.write(img) + temp_file.close() has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins @@ -454,21 +359,20 @@ class Plugin(Base): if Ocr and not forceUser: sleep(randint(3000, 5000) / 1000.0) - if self.pyfile.abort: - self.abort() + if self.pyfile.abort: raise Abort ocr = Ocr() - result = ocr.get_captcha(tmpCaptcha.name) + result = ocr.get_captcha(temp_file.name) else: captchaManager = self.core.captchaManager - task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type) + task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) self.cTask = task - captchaManager.handleCaptcha(task, timeout) + captchaManager.handleCaptcha(task) while task.isWaiting(): if self.pyfile.abort: captchaManager.removeTask(task) - self.abort() + raise Abort sleep(1) captchaManager.removeTask(task) @@ -478,21 +382,21 @@ class Plugin(Base): elif task.error: self.fail(task.error) elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins")) + self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) result = task.result - self.logDebug("Received captcha result: " + str(result)) + self.log.debug("Received captcha result: %s" % str(result)) if not self.core.debug: try: - remove(tmpCaptcha.name) + remove(temp_file.name) except: pass return result - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True): + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """Load content at url and returns it :param url: @@ -500,44 +404,35 @@ class Plugin(Base): :param post: :param ref: :param cookies: - :param just_header: If True only the header will be retrieved and returned as dict + :param just_header: if True only the header will be retrieved and returned as dict :param decode: Wether to decode the output according to http header, should be True in most cases - :param follow_location: If True follow location else not - :param save_cookies: If True saves received cookies else discard them :return: Loaded content """ - if self.pyfile.abort: - self.abort() - - if not url: - self.fail(_("No url given")) - - if type(url) == unicode: # utf8 vs decode -> please use decode attribute in all future plugins - url = str(url) #: encode('utf8') - - url = url.strip() - - if self.core.debug: - self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) + if self.pyfile.abort: raise Abort + #utf8 vs decode -> please use decode attribute in all future plugins + if type(url) == unicode: url = str(url) - res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies) + res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.core.debug: from inspect import currentframe frame = currentframe() - framefile = save_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + if not exists(join("tmp", self.__name__)): + makedirs(join("tmp", self.__name__)) + + f = open( + join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + , "wb") + del frame # delete the frame or it wont be cleaned + try: - if not exists(join("tmp", self.__name__)): - makedirs(join("tmp", self.__name__)) + tmp = res.encode("utf8") + except: + tmp = res - with open(framefile, "wb") as f: - del frame #: delete the frame or it wont be cleaned - if decode: - res = res.encode('utf-8') - f.write(res) - except IOError, e: - self.logError(e) + f.write(tmp) + f.close() if just_header: #parse header @@ -547,7 +442,7 @@ class Plugin(Base): if not line or ":" not in line: continue key, none, value = line.partition(":") - key = key.strip().lower() + key = key.lower().strip() value = value.strip() if key in header: @@ -561,7 +456,6 @@ class Plugin(Base): return res - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder @@ -574,19 +468,6 @@ class Plugin(Base): the filename will be changed if needed :return: The location where the file was saved """ - if self.pyfile.abort: - self.abort() - - if not url: - self.fail(_("No url given")) - - if type(url) == unicode: - url = str(url) - - url = url.strip() - - if self.core.debug: - self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")]) self.checkForSameFiles() @@ -597,16 +478,16 @@ class Plugin(Base): location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): - try: - makedirs(location, int(self.core.config['permission']['folder'], 8)) + makedirs(location, int(self.core.config["permission"]["folder"], 8)) - if self.core.config['permission']['change_dl'] and os.name != "nt": - uid = getpwnam(self.config['permission']['user'])[2] - gid = getgrnam(self.config['permission']['group'])[2] - chown(location, uid, gid) + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] - except Exception, e: - self.fail(e) + chown(location, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) # convert back to unicode location = fs_decode(location) @@ -623,38 +504,31 @@ class Plugin(Base): finally: self.pyfile.size = self.req.size - if newname: - newname = urlparse(newname).path.split("/")[-1] - - if disposition and newname != name: - self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname}) - self.pyfile.name = newname - filename = join(location, newname) + if disposition and newname and newname != name: #triple check, just to be sure + self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = join(location, newname) fs_filename = fs_encode(filename) - if self.core.config['permission']['change_file']: - try: - chmod(fs_filename, int(self.core.config['permission']['file'], 8)) - except Exception, e: - self.logWarning(_("Setting file mode failed"), e) + if self.core.config["permission"]["change_file"]: + chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) - if self.core.config['permission']['change_dl'] and os.name != "nt": + if self.core.config["permission"]["change_dl"] and os.name != "nt": try: - uid = getpwnam(self.config['permission']['user'])[2] - gid = getgrnam(self.config['permission']['group'])[2] - chown(fs_filename, uid, gid) + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + chown(fs_filename, uid, gid) except Exception, e: - self.logWarning(_("Setting User and Group failed"), e) + self.log.warning(_("Setting User and Group failed: %s") % str(e)) self.lastDownload = filename return self.lastDownload - def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): """ checks the content of the last downloaded file, re match is saved to `lastCheck` - + :param rules: dict with names and rules to match (compiled regexp or strings) :param api_size: expected file size :param max_size: if the file is larger then it wont be checked @@ -663,21 +537,19 @@ class Plugin(Base): :return: dictionary key of the first rule that matched """ lastDownload = fs_encode(self.lastDownload) - if not exists(lastDownload): - return None + if not exists(lastDownload): return None size = stat(lastDownload) size = size.st_size if api_size and api_size <= size: return None elif size > max_size and not read_size: return None - self.logDebug("Download Check triggered") - - with open(lastDownload, "rb") as f: - content = f.read(read_size if read_size else -1) - + self.log.debug("Download Check triggered") + f = open(lastDownload, "rb") + content = f.read(read_size if read_size else -1) + f.close() #produces encoding errors, better log to other file in the future? - #self.logDebug("Content: %s" % content) + #self.log.debug("Content: %s" % content) for name, rule in rules.iteritems(): if type(rule) in (str, unicode): if rule in content: @@ -723,27 +595,23 @@ class Plugin(Base): if starting and self.core.config['download']['skip_existing'] and exists(location): size = os.stat(location).st_size if size >= self.pyfile.size: - raise SkipDownload("File exists") + raise SkipDownload("File exists.") pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) if pyfile: if exists(location): raise SkipDownload(pyfile[0]) - self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name) - + self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) def clean(self): """ clean everything and remove references """ if hasattr(self, "pyfile"): del self.pyfile - if hasattr(self, "req"): self.req.close() del self.req - if hasattr(self, "thread"): del self.thread - if hasattr(self, "html"): del self.html -- cgit v1.2.3