diff options
Diffstat (limited to 'module/plugins')
| -rw-r--r-- | module/plugins/Account.py | 7 | ||||
| -rw-r--r-- | module/plugins/AccountManager.py | 10 | ||||
| -rw-r--r-- | module/plugins/Base.py | 101 | ||||
| -rw-r--r-- | module/plugins/Container.py | 75 | ||||
| -rw-r--r-- | module/plugins/Crypter.py | 250 | ||||
| -rw-r--r-- | module/plugins/Hook.py | 2 | ||||
| -rw-r--r-- | module/plugins/Hoster.py | 444 | ||||
| -rw-r--r-- | module/plugins/PluginManager.py | 52 | ||||
| -rw-r--r-- | module/plugins/container/CCF.py | 4 | ||||
| -rw-r--r-- | module/plugins/container/LinkList.py | 4 | ||||
| -rw-r--r-- | module/plugins/container/RSDF.py | 4 | ||||
| -rw-r--r-- | module/plugins/hooks/UpdateManager.py | 5 |
12 files changed, 764 insertions, 194 deletions
diff --git a/module/plugins/Account.py b/module/plugins/Account.py index 86b73c99c..6b65051db 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -149,6 +149,13 @@ class Account(Base, AccountInfo): def getAccountRequest(self): return self.core.requestFactory.getRequest(self.__name__, self.cj) + def getDownloadSettings(self): + """ Can be overwritten to change download settings. Default is no chunkLimit, multiDL, resumeDownload + + :return: (chunkLimit, multiDL, resumeDownload) / (int,bool,bool) + """ + return -1, True, True + @lock def getAccountInfo(self, force=False): """retrieve account infos for an user, do **not** overwrite this method!\\ diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index c718510ed..77139206c 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -21,7 +21,6 @@ from threading import Lock from random import choice from module.common.json_layer import json -from module.interaction.PullEvents import AccountUpdateEvent from module.utils import lock class AccountManager(): @@ -85,12 +84,15 @@ class AccountManager(): self.createAccount(plugin, user, password, options) self.saveAccounts() + self.sendChange() + @lock def removeAccount(self, plugin, user): """remove account""" if plugin in self.accounts and user in self.accounts[plugin]: del self.accounts[plugin][user] self.core.db.removeAccount(plugin, user) + self.sendChange() else: self.core.log.debug("Remove non existing account %s %s" % (plugin, user)) @@ -118,9 +120,6 @@ class AccountManager(): for acc in p_dict.itervalues(): acc.getAccountInfo() - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) - return self.accounts def refreshAllAccounts(self): @@ -131,5 +130,4 @@ class AccountManager(): def sendChange(self): - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) + self.core.eventManager.dispatchEvent("accountsUpdated")
\ No newline at end of file diff --git a/module/plugins/Base.py b/module/plugins/Base.py index 36df7e423..b2338a01f 100644 --- a/module/plugins/Base.py +++ b/module/plugins/Base.py @@ -18,12 +18,19 @@ """ import sys +from module.utils.fs import exists, makedirs, join -# TODO: config format definition +# TODO # more attributes if needed # get rid of catpcha & container plugins ?! (move to crypter & internals) # adapt old plugins as needed +class Fail(Exception): + """ raised when failed """ + +class Retry(Exception): + """ raised when start again from beginning """ + class Base(object): """ The Base plugin class with all shared methods and every possible attribute for plugin definition. @@ -31,7 +38,8 @@ class Base(object): __version__ = "0.1" #: Regexp pattern which will be matched for download plugins __pattern__ = r"" - #: Flat config definition + #: Config definition: list of (name, type, verbose_name, default_value) or + #: (name, type, verbose_name, short_description, default_value) __config__ = tuple() #: Short description, one liner __description__ = "" @@ -41,7 +49,7 @@ class Base(object): __dependencies__ = tuple() #: Tags to categorize the plugin __tags__ = tuple() - #: Base64 encoded .png icon + #: Base64 encoded .png icon, please don't use sizes above ~3KB __icon__ = "" #: Alternative, link to png icon __icon_url__ = "" @@ -62,18 +70,25 @@ class Base(object): self.config = core.config #log functions - def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + def logInfo(self, *args, **kwargs): + self._log("info", *args, **kwargs) + + def logWarning(self, *args, **kwargs): + self._log("warning", *args, **kwargs) - def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + def logError(self, *args, **kwargs): + self._log("error", *args, **kwargs) - def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + def logDebug(self, *args, **kwargs): + self._log("debug", *args, **kwargs) - def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) + def _log(self, level, *args, **kwargs): + if "sep" in kwargs: + sep = "%s" % kwargs["sep"] + else: + sep = " | " + getattr(self.log, level)("%s: %s" % (self.__name__, sep.join([a if isinstance(a, basestring) else str(a) for a in args]))) def setConf(self, option, value): """ see `setConfig` """ @@ -129,3 +144,67 @@ class Base(object): #noinspection PyUnresolvedReferences sys.stdout = sys._stdout embed() + + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): + """Load content at url and returns it + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param just_header: if True only the header will be retrieved and returned as dict + :param decode: Wether to decode the output according to http header, should be True in most cases + :return: Loaded content + """ + if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") + + if type(url) == unicode: url = str(url) + + res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) + + if self.core.debug: + from inspect import currentframe + + frame = currentframe() + if not exists(join("tmp", self.__name__)): + makedirs(join("tmp", self.__name__)) + + f = open( + join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + , "wb") + del frame # delete the frame or it wont be cleaned + + try: + tmp = res.encode("utf8") + except: + tmp = res + + f.write(tmp) + f.close() + + if just_header: + #parse header + header = {"code": self.req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: continue + + key, none, value = line.partition(":") + key = key.lower().strip() + value = value.strip() + + if key in header: + if type(header[key]) == list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value + res = header + + return res + + def fail(self, reason): + """ fail and give reason """ + raise Fail(reason)
\ No newline at end of file diff --git a/module/plugins/Container.py b/module/plugins/Container.py deleted file mode 100644 index c233d3710..000000000 --- a/module/plugins/Container.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Crypter import Crypter - -from os.path import join, exists, basename -from os import remove -import re - -class Container(Crypter): - __name__ = "Container" - __version__ = "0.1" - __pattern__ = None - __type__ = "container" - __description__ = """Base container plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - - def preprocessing(self, thread): - """prepare""" - - self.setup() - self.thread = thread - - self.loadToDisk() - - self.decrypt(self.pyfile) - self.deleteTmp() - - self.createPackages() - - - def loadToDisk(self): - """loads container to disk if its stored remotely and overwrite url, - or check existent on several places at disk""" - - if self.pyfile.url.startswith("http"): - self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] - content = self.load(self.pyfile.url) - self.pyfile.url = join(self.config["general"]["download_folder"], self.pyfile.name) - f = open(self.pyfile.url, "wb" ) - f.write(content) - f.close() - - else: - self.pyfile.name = basename(self.pyfile.url) - if not exists(self.pyfile.url): - if exists(join(pypath, self.pyfile.url)): - self.pyfile.url = join(pypath, self.pyfile.url) - else: - self.fail(_("File not exists.")) - - - def deleteTmp(self): - if self.pyfile.name.startswith("tmp_"): - remove(self.pyfile.url) - - diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index d1549fe80..fc54b32d7 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -1,72 +1,214 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Plugin import Plugin - -class Crypter(Plugin): - __name__ = "Crypter" - __version__ = "0.1" - __pattern__ = None - __type__ = "container" - __description__ = """Base crypter plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def __init__(self, pyfile): - Plugin.__init__(self, pyfile) - - #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) - self.packages = [] +from module.Api import Destination +from module.common.packagetools import parseNames +from module.utils import to_list +from module.utils.fs import exists + +from Base import Base, Retry + +class Package: + """ Container that indicates new package should be created """ + def __init__(self, name, urls=None, dest=Destination.Queue): + self.name = name, + self.urls = urls if urls else [] + self.dest = dest + + def addUrl(self, url): + self.urls.append(url) + +class PyFileMockup: + """ Legacy class needed by old crypter plugins """ + def __init__(self, url): + self.url = url + self.name = url + +class Crypter(Base): + """ + Base class for (de)crypter plugins. Overwrite decrypt* methods. + + How to use decrypt* methods + --------------------------- + + You have to overwrite at least one method of decryptURL, decryptURLs, decryptFile. + + After decrypting and generating urls/packages you have to return the result at the\ + end of your method. Valid return Data is: + + `Package` instance + A **new** package will be created with the name and the urls of the object. + + List of urls and `Package` instances + All urls in the list will be added to the **current** package. For each `Package`\ + instance a new package will be created. + + """ - #: List of urls, pyLoad will generate packagenames + @classmethod + def decrypt(cls, core, url_or_urls): + """Static method to decrypt, something. Can be used by other plugins. + + :param core: pyLoad `Core`, needed in decrypt context + :param url_or_urls: List of urls or urls + :return: List of decrypted urls, all packages info removed + """ + urls = to_list(url_or_urls) + p = cls(core) + try: + result = p.processDecrypt(urls) + finally: + p.clean() + + ret = [] + + for url_or_pack in result: + if isinstance(url_or_pack, Package): #package + ret.extend(url_or_pack.urls) + else: # single url + ret.append(url_or_pack) + + return ret + + def __init__(self, core, pid=-1, password=None): + Base.__init__(self, core) + self.req = core.requestFactory.getRequest(self.__name__) + + # Package id plugin was initilized for, dont use this, its not guaranteed to be set + self.pid = pid + + #: Password supplied by user + self.password = password + + # For old style decrypter, do not use these ! + self.packages = [] self.urls = [] - - self.multiDL = True - self.limitDL = 0 - - - def preprocessing(self, thread): - """prepare""" - self.setup() - self.thread = thread - - self.decrypt(self.pyfile) - - self.createPackages() - - - def decrypt(self, pyfile): + self.pyfile = None + + self.init() + + def init(self): + """More init stuff if needed""" + + def setup(self): + """Called everytime before decrypting. A Crypter plugin will be most likly used for several jobs.""" + + def decryptURL(self, url): + """Decrypt a single url + + :param url: url to decrypt + :return: See `Crypter` Documentation + """ + raise NotImplementedError + + def decryptURLs(self, urls): + """Decrypt a bunch of urls + + :param urls: list of urls + :return: See `Crypter` Documentation + """ raise NotImplementedError + def decryptFile(self, content): + """Decrypt file content + + :param content: content to decrypt as string + :return: See `Crypter Documentation + """ + raise NotImplementedError + + def generatePackages(self, urls): + """Generates `Package` instances and names from urls. Usefull for many different link and no\ + given package name. + + :param urls: list of urls + :return: list of `Package` + """ + return [Package(name, purls) for name, purls in parseNames([(url,url) for url in urls]).iteritems()] + + def processDecrypt(self, urls): + """ Internal method to select decrypting method + + :param urls: List of urls/content + :return: + """ + cls = self.__class__ + + # seperate local and remote files + content, urls = self.getLocalContent(urls) + + if hasattr(cls, "decryptURLs"): + self.setup() + result = to_list(self.decryptURLs(urls)) + elif hasattr(cls, "decryptURL"): + result = [] + for url in urls: + self.setup() + result.extend(to_list(self.decryptURL(url))) + elif hasattr(cls, "decrypt"): + self.logDebug("Deprecated .decrypt() method in Crypter plugin") + result = [] # TODO + else: + self.logError("No Decrypting method was overwritten") + result = [] + + if hasattr(cls, "decryptFile"): + for c in content: + self.setup() + result.extend(to_list(self.decryptFile(c))) + + return result + + def getLocalContent(self, urls): + """Load files from disk + + :param urls: + :return: content, remote urls + """ + content = [] + # do nothing if no decryptFile method + if hasattr(self.__class__, "decryptFile"): + remote = [] + for url in urls: + path = None + if url.startswith("http"): + path = None # skip directly + elif exists(url): + path = url + elif exists(self.core.path(url)): + path = self.core.path(url) + + if path: + f = open(path, "wb") + content.append(f.read()) + f.close() + else: + remote.append(url) + + #swap filtered url list + urls = remote + + return content, urls + + def retry(self): + """ Retry decrypting, will only work once. Somewhat deprecated method, should be avoided. """ + raise Retry() + def createPackages(self): - """ create new packages from self.packages """ + """ Deprecated """ + self.logDebug("Deprecated method .createPackages()") for pack in self.packages: self.log.debug("Parsed package %(name)s with %(len)d links" % { "name" : pack[0], "len" : len(pack[1]) } ) links = [x.decode("utf-8") for x in pack[1]] - pid = self.core.api.addPackage(pack[0], links, self.pyfile.package().queue) + pid = self.core.api.files.addLinks(self.pid, links) - if self.pyfile.package().password: - self.core.api.setPackageData(pid, {"password": self.pyfile.package().password}) if self.urls: self.core.api.generateAndAddPackages(self.urls) + def clean(self): + if hasattr(self, "req"): + self.req.close() + del self.req
\ No newline at end of file diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index 860dc76bb..a3b86a794 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -20,7 +20,7 @@ from traceback import print_exc -from Plugin import Base +from Base import Base class Expose(object): """ used for decoration to declare rpc services """ diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index aa50099fb..54c2efdfd 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -13,13 +13,39 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay + + @author: RaNaN, spoob, mkaay """ -from module.plugins.Plugin import Plugin +from time import time, sleep +from random import randint + +import os + +if os.name != "nt": + from module.utils.fs import chown + from pwd import getpwnam + from grp import getgrnam + +from Base import Base, Fail, Retry +from module.utils import chunks #legacy import +from module.utils.fs import save_join, save_path, fs_encode, fs_decode,\ + remove, makedirs, chmod, stat, exists, join + + +class Abort(Exception): + """ raised when aborted """ + +class Reconnect(Exception): + """ raised when reconnected """ -class Hoster(Plugin): +class SkipDownload(Exception): + """ raised when download should be skipped """ + +class Hoster(Base): + """ + Base plugin for hoster plugin. Overwrite getInfo for online status retrieval, process for downloading. + """ @staticmethod def getInfo(urls): @@ -28,6 +54,412 @@ class Hoster(Plugin): where status is one of API pyfile statusses. :param urls: List of urls - :return: + :return: yield list of tuple with results (name, size, status, url) + """ + pass + + def __init__(self, pyfile): + Base.__init__(self, pyfile.m.core) + + self.wantReconnect = False + #: enables simultaneous processing of multiple downloads + self.multiDL = True + self.limitDL = 0 + #: chunk limit + self.chunkLimit = 1 + #: enables resume (will be ignored if server dont accept chunks) + self.resumeDownload = False + + #: time() + wait in seconds + self.waitUntil = 0 + self.waiting = False + + self.ocr = None #captcha reader instance + #: account handler instance, see :py:class:`Account` + self.account = self.core.accountManager.getAccountForPlugin(self.__name__) + + #: premium status + self.premium = False + #: username/login + self.user = None + + if self.account and not self.account.isUsable(): self.account = None + if self.account: + self.user = self.account.loginname + #: Browser instance, see `network.Browser` + self.req = self.account.getAccountRequest() + # Default: -1, True, True + self.chunkLimit, self.resumeDownload, self.multiDL = self.account.getDownloadSettings() + self.premium = self.account.isPremium() + else: + self.req = self.core.requestFactory.getRequest(self.__name__) + + #: associated pyfile instance, see `PyFile` + self.pyfile = pyfile + self.thread = None # holds thread in future + + #: location where the last call to download was saved + self.lastDownload = "" + #: re match of the last call to `checkDownload` + self.lastCheck = None + #: js engine, see `JsEngine` + self.js = self.core.js + self.cTask = None #captcha task + + self.retries = 0 # amount of retries already made + self.html = None # some plugins store html code here + + self.init() + + def getChunkCount(self): + if self.chunkLimit <= 0: + return self.config["download"]["chunks"] + return min(self.config["download"]["chunks"], self.chunkLimit) + + def __call__(self): + return self.__name__ + + def init(self): + """initialize the plugin (in addition to `__init__`)""" + pass + + def setup(self): + """ setup for enviroment and other things, called before downloading (possibly more than one time)""" + pass + + def preprocessing(self, thread): + """ handles important things to do before starting """ + self.thread = thread + + if self.account: + # will force a relogin or reload of account info if necessary + self.account.getAccountInfo() + else: + self.req.clearCookies() + + self.setup() + + self.pyfile.setStatus("starting") + + return self.process(self.pyfile) + + + def process(self, pyfile): + """the 'main' method of every plugin, you **have to** overwrite it""" + raise NotImplementedError + + def resetAccount(self): + """ dont use account and retry download """ + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retry() + + def checksum(self, local_file=None): + """ + return codes: + 0 - checksum ok + 1 - checksum wrong + 5 - can't get checksum + 10 - not implemented + 20 - unknown error + """ + #@TODO checksum check hook + + return True, 10 + + + def setWait(self, seconds, reconnect=False): + """Set a specific wait time later used with `wait` + + :param seconds: wait time in seconds + :param reconnect: True if a reconnect would avoid wait time + """ + if reconnect: + self.wantReconnect = True + self.pyfile.waitUntil = time() + int(seconds) + + def wait(self): + """ waits the time previously set """ + self.waiting = True + self.pyfile.setStatus("waiting") + + while self.pyfile.waitUntil > time(): + self.thread.m.reconnecting.wait(2) + + if self.pyfile.abort: raise Abort + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + self.waiting = False + self.pyfile.setStatus("starting") + + def offline(self): + """ fail and indicate file is offline """ + raise Fail("offline") + + def tempOffline(self): + """ fail and indicates file ist temporary offline, the core may take consequences """ + raise Fail("temp. offline") + + def retry(self, max_tries=3, wait_time=1, reason=""): + """Retries and begin again from the beginning + + :param max_tries: number of maximum retries + :param wait_time: time to wait in seconds + :param reason: reason for retrying, will be passed to fail if max_tries reached + """ + if 0 < max_tries <= self.retries: + if not reason: reason = "Max retries reached" + raise Fail(reason) + + self.wantReconnect = False + self.setWait(wait_time) + self.wait() + + self.retries += 1 + raise Retry(reason) + + def invalidCaptcha(self): + if self.cTask: + self.cTask.invalid() + + def correctCaptcha(self): + if self.cTask: + self.cTask.correct() + + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', + result_type='textual'): + """ Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param forceUser: if True, ocr is not used + :param imgtype: Type of the Image + :param result_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + + :return: result of decrypting + """ + + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time())[-6:].replace(".", "") + temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + temp_file.write(img) + temp_file.close() + + has_plugin = self.__name__ in self.core.pluginManager.getPlugins("captcha") + + if self.core.captcha: + Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) + else: + Ocr = None + + if Ocr and not forceUser: + sleep(randint(3000, 5000) / 1000.0) + if self.pyfile.abort: raise Abort + + ocr = Ocr() + result = ocr.get_captcha(temp_file.name) + else: + captchaManager = self.core.captchaManager + task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) + self.cTask = task + captchaManager.handleCaptcha(task) + + while task.isWaiting(): + if self.pyfile.abort: + captchaManager.removeTask(task) + raise Abort + sleep(1) + + captchaManager.removeTask(task) + + if task.error and has_plugin: #ignore default error message since the user could use OCR + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) + + result = task.result + self.log.debug("Received captcha result: %s" % str(result)) + + if not self.core.debug: + try: + remove(temp_file.name) + except: + pass + + return result + + + def load(self, *args, **kwargs): + """ See 'Base' load method for more info """ + if self.pyfile.abort: raise Abort + return Base.load(self, *args, **kwargs) + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + """Downloads the content at url to download folder + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param disposition: if True and server provides content-disposition header\ + the filename will be changed if needed + :return: The location where the file was saved + """ + + self.checkForSameFiles() + + self.pyfile.setStatus("downloading") + + download_folder = self.config['general']['download_folder'] + + location = save_join(download_folder, self.pyfile.package().folder) + + if not exists(location): + makedirs(location, int(self.core.config["permission"]["folder"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(location, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + # convert back to unicode + location = fs_decode(location) + name = save_path(self.pyfile.name) + + filename = join(location, name) + + self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) + + try: + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, + chunks=self.getChunkCount(), resume=self.resumeDownload, + progressNotify=self.pyfile.setProgress, disposition=disposition) + finally: + self.pyfile.size = self.req.size + + if disposition and newname and newname != name: #triple check, just to be sure + self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = join(location, newname) + + fs_filename = fs_encode(filename) + + if self.core.config["permission"]["change_file"]: + chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(fs_filename, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + self.lastDownload = filename + return self.lastDownload + + def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): + """ checks the content of the last downloaded file, re match is saved to `lastCheck` + + :param rules: dict with names and rules to match (compiled regexp or strings) + :param api_size: expected file size + :param max_size: if the file is larger then it wont be checked + :param delete: delete if matched + :param read_size: amount of bytes to read from files larger then max_size + :return: dictionary key of the first rule that matched + """ + lastDownload = fs_encode(self.lastDownload) + if not exists(lastDownload): return None + + size = stat(lastDownload) + size = size.st_size + + if api_size and api_size <= size: return None + elif size > max_size and not read_size: return None + self.log.debug("Download Check triggered") + f = open(lastDownload, "rb") + content = f.read(read_size if read_size else -1) + f.close() + #produces encoding errors, better log to other file in the future? + #self.log.debug("Content: %s" % content) + for name, rule in rules.iteritems(): + if type(rule) in (str, unicode): + if rule in content: + if delete: + remove(lastDownload) + return name + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + if delete: + remove(lastDownload) + self.lastCheck = m + return name + + + def getPassword(self): + """ get the password the user provided in the package""" + password = self.pyfile.package().password + if not password: return "" + return password + + + def checkForSameFiles(self, starting=False): + """ checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises SkipDownload: """ - pass
\ No newline at end of file + + pack = self.pyfile.package() + + for pyfile in self.core.files.cache.values(): + if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile.status in (0, 12): #finished or downloading + raise SkipDownload(pyfile.pluginname) + elif pyfile.status in ( + 5, 7) and starting: #a download is waiting/starting and was appenrently started before + raise SkipDownload(pyfile.pluginname) + + download_folder = self.config['general']['download_folder'] + location = save_join(download_folder, pack.folder, self.pyfile.name) + + if starting and self.core.config['download']['skip_existing'] and exists(location): + size = os.stat(location).st_size + if size >= self.pyfile.size: + raise SkipDownload("File exists.") + + pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + if pyfile: + if exists(location): + raise SkipDownload(pyfile[0]) + + self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) + + def clean(self): + """ clean everything and remove references """ + if hasattr(self, "pyfile"): + del self.pyfile + if hasattr(self, "req"): + self.req.close() + del self.req + if hasattr(self, "thread"): + del self.thread + if hasattr(self, "html"): + del self.html diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index 18dea7699..e00c1e1f5 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -42,7 +42,7 @@ PluginTuple = namedtuple("PluginTuple", "version re deps user path") class PluginManager: ROOT = "module.plugins." USERROOT = "userplugins." - TYPES = ("crypter", "container", "hoster", "captcha", "accounts", "hooks", "internal") + TYPES = ("crypter", "hoster", "captcha", "accounts", "hooks", "internal") SINGLE = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?<!")"(?!")|\'|\().*(?:(?<!")"(?!")|\'|\)))', re.I) @@ -216,54 +216,54 @@ class PluginManager: def parseUrls(self, urls): - """parse plugins for given list of urls""" + """parse plugins for given list of urls, seperate to crypter and hoster""" - res = [] # tupels of (url, plugin) + res = {"hoster": [], "crypter": []} # tupels of (url, plugin) for url in urls: if type(url) not in (str, unicode, buffer): self.log.debug("Parsing invalid type %s" % type(url)) continue + found = False for ptype, name in self.history: if self.plugins[ptype][name].re.match(url): - res.append((url, name)) + res[ptype].append((url, name)) found = (ptype, name) + break - if found and self.history[0] != found: - # found match, update history - self.history.remove(found) - self.history.insert(0, found) + if found: # found match + if self.history[0] != found: #update history + self.history.remove(found) + self.history.insert(0, found) continue - for ptype in ("crypter", "hoster", "container"): + for ptype in ("crypter", "hoster"): for name, plugin in self.plugins[ptype].iteritems(): if plugin.re.match(url): - res.append((url, name)) + res[ptype].append((url, name)) self.history.insert(0, (ptype, name)) del self.history[10:] # cut down to size of 10 found = True break if not found: - res.append((url, "BasePlugin")) + res["hoster"].append((url, "BasePlugin")) - return res + return res["hoster"], res["crypter"] def getPlugins(self, type): - # TODO clean this workaround - if type not in self.plugins: type += "s" # append s, so updater can find the plugins - return self.plugins[type] + return self.plugins.get(type, None) - def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): + def findPlugin(self, name, pluginlist=("hoster", "crypter")): for ptype in pluginlist: if name in self.plugins[ptype]: return ptype, self.plugins[ptype][name] return None, None def getPlugin(self, name, original=False): - """return plugin module from hoster|decrypter|container""" + """return plugin module from hoster|decrypter""" type, plugin = self.findPlugin(name) if not plugin: @@ -412,22 +412,4 @@ class PluginManager: :return: List of unfullfilled dependencies """ pass - - -if __name__ == "__main__": - _ = lambda x: x - pypath = "/home/christian/Projekte/pyload-0.4/module/plugins" - - from time import time - - p = PluginManager(None) - - a = time() - - test = ["http://www.youtube.com/watch?v=%s" % x for x in range(0, 100)] - print p.parseUrls(test) - - b = time() - - print b - a, "s" diff --git a/module/plugins/container/CCF.py b/module/plugins/container/CCF.py index 301b033d4..ab7ff1099 100644 --- a/module/plugins/container/CCF.py +++ b/module/plugins/container/CCF.py @@ -4,13 +4,13 @@ import re from urllib2 import build_opener -from module.plugins.Container import Container +from module.plugins.Crypter import Crypter from module.lib.MultipartPostHandler import MultipartPostHandler from os import makedirs from os.path import exists, join -class CCF(Container): +class CCF(Crypter): __name__ = "CCF" __version__ = "0.2" __pattern__ = r"(?!http://).*\.ccf$" diff --git a/module/plugins/container/LinkList.py b/module/plugins/container/LinkList.py index b9eb4b972..614c76c90 100644 --- a/module/plugins/container/LinkList.py +++ b/module/plugins/container/LinkList.py @@ -2,9 +2,9 @@ # -*- coding: utf-8 -*- -from module.plugins.Container import Container +from module.plugins.Crypter import Crypter -class LinkList(Container): +class LinkList(Crypter): __name__ = "LinkList" __version__ = "0.11" __pattern__ = r".+\.txt$" diff --git a/module/plugins/container/RSDF.py b/module/plugins/container/RSDF.py index ea5cd67f2..cbc9864b1 100644 --- a/module/plugins/container/RSDF.py +++ b/module/plugins/container/RSDF.py @@ -5,9 +5,9 @@ import base64 import binascii import re -from module.plugins.Container import Container +from module.plugins.Crypter import Crypter -class RSDF(Container): +class RSDF(Crypter): __name__ = "RSDF" __version__ = "0.21" __pattern__ = r".*\.rsdf" diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py index 4324a96ba..d0c7f213d 100644 --- a/module/plugins/hooks/UpdateManager.py +++ b/module/plugins/hooks/UpdateManager.py @@ -61,6 +61,11 @@ class UpdateManager(Hook): @threaded def periodical(self): + + if self.core.version.endswith("-dev"): + self.logDebug("No update check performed on dev version.") + return + update = self.checkForUpdate() if update: self.info["pyload"] = True |