From d35c003cc53d4723d1dfe0d81eeb9bea78cee594 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 31 Dec 2011 16:01:24 +0100 Subject: new crypter plugin API, now decrypting possible for now. --- module/plugins/Crypter.py | 250 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 196 insertions(+), 54 deletions(-) (limited to 'module/plugins/Crypter.py') diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index d1549fe80..fc54b32d7 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -1,72 +1,214 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from module.plugins.Plugin import Plugin - -class Crypter(Plugin): - __name__ = "Crypter" - __version__ = "0.1" - __pattern__ = None - __type__ = "container" - __description__ = """Base crypter plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def __init__(self, pyfile): - Plugin.__init__(self, pyfile) - - #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) - self.packages = [] +from module.Api import Destination +from module.common.packagetools import parseNames +from module.utils import to_list +from module.utils.fs import exists + +from Base import Base, Retry + +class Package: + """ Container that indicates new package should be created """ + def __init__(self, name, urls=None, dest=Destination.Queue): + self.name = name, + self.urls = urls if urls else [] + self.dest = dest + + def addUrl(self, url): + self.urls.append(url) + +class PyFileMockup: + """ Legacy class needed by old crypter plugins """ + def __init__(self, url): + self.url = url + self.name = url + +class Crypter(Base): + """ + Base class for (de)crypter plugins. Overwrite decrypt* methods. + + How to use decrypt* methods + --------------------------- + + You have to overwrite at least one method of decryptURL, decryptURLs, decryptFile. + + After decrypting and generating urls/packages you have to return the result at the\ + end of your method. Valid return Data is: + + `Package` instance + A **new** package will be created with the name and the urls of the object. + + List of urls and `Package` instances + All urls in the list will be added to the **current** package. For each `Package`\ + instance a new package will be created. + + """ - #: List of urls, pyLoad will generate packagenames + @classmethod + def decrypt(cls, core, url_or_urls): + """Static method to decrypt, something. Can be used by other plugins. + + :param core: pyLoad `Core`, needed in decrypt context + :param url_or_urls: List of urls or urls + :return: List of decrypted urls, all packages info removed + """ + urls = to_list(url_or_urls) + p = cls(core) + try: + result = p.processDecrypt(urls) + finally: + p.clean() + + ret = [] + + for url_or_pack in result: + if isinstance(url_or_pack, Package): #package + ret.extend(url_or_pack.urls) + else: # single url + ret.append(url_or_pack) + + return ret + + def __init__(self, core, pid=-1, password=None): + Base.__init__(self, core) + self.req = core.requestFactory.getRequest(self.__name__) + + # Package id plugin was initilized for, dont use this, its not guaranteed to be set + self.pid = pid + + #: Password supplied by user + self.password = password + + # For old style decrypter, do not use these ! + self.packages = [] self.urls = [] - - self.multiDL = True - self.limitDL = 0 - - - def preprocessing(self, thread): - """prepare""" - self.setup() - self.thread = thread - - self.decrypt(self.pyfile) - - self.createPackages() - - - def decrypt(self, pyfile): + self.pyfile = None + + self.init() + + def init(self): + """More init stuff if needed""" + + def setup(self): + """Called everytime before decrypting. A Crypter plugin will be most likly used for several jobs.""" + + def decryptURL(self, url): + """Decrypt a single url + + :param url: url to decrypt + :return: See `Crypter` Documentation + """ + raise NotImplementedError + + def decryptURLs(self, urls): + """Decrypt a bunch of urls + + :param urls: list of urls + :return: See `Crypter` Documentation + """ raise NotImplementedError + def decryptFile(self, content): + """Decrypt file content + + :param content: content to decrypt as string + :return: See `Crypter Documentation + """ + raise NotImplementedError + + def generatePackages(self, urls): + """Generates `Package` instances and names from urls. Usefull for many different link and no\ + given package name. + + :param urls: list of urls + :return: list of `Package` + """ + return [Package(name, purls) for name, purls in parseNames([(url,url) for url in urls]).iteritems()] + + def processDecrypt(self, urls): + """ Internal method to select decrypting method + + :param urls: List of urls/content + :return: + """ + cls = self.__class__ + + # seperate local and remote files + content, urls = self.getLocalContent(urls) + + if hasattr(cls, "decryptURLs"): + self.setup() + result = to_list(self.decryptURLs(urls)) + elif hasattr(cls, "decryptURL"): + result = [] + for url in urls: + self.setup() + result.extend(to_list(self.decryptURL(url))) + elif hasattr(cls, "decrypt"): + self.logDebug("Deprecated .decrypt() method in Crypter plugin") + result = [] # TODO + else: + self.logError("No Decrypting method was overwritten") + result = [] + + if hasattr(cls, "decryptFile"): + for c in content: + self.setup() + result.extend(to_list(self.decryptFile(c))) + + return result + + def getLocalContent(self, urls): + """Load files from disk + + :param urls: + :return: content, remote urls + """ + content = [] + # do nothing if no decryptFile method + if hasattr(self.__class__, "decryptFile"): + remote = [] + for url in urls: + path = None + if url.startswith("http"): + path = None # skip directly + elif exists(url): + path = url + elif exists(self.core.path(url)): + path = self.core.path(url) + + if path: + f = open(path, "wb") + content.append(f.read()) + f.close() + else: + remote.append(url) + + #swap filtered url list + urls = remote + + return content, urls + + def retry(self): + """ Retry decrypting, will only work once. Somewhat deprecated method, should be avoided. """ + raise Retry() + def createPackages(self): - """ create new packages from self.packages """ + """ Deprecated """ + self.logDebug("Deprecated method .createPackages()") for pack in self.packages: self.log.debug("Parsed package %(name)s with %(len)d links" % { "name" : pack[0], "len" : len(pack[1]) } ) links = [x.decode("utf-8") for x in pack[1]] - pid = self.core.api.addPackage(pack[0], links, self.pyfile.package().queue) + pid = self.core.api.files.addLinks(self.pid, links) - if self.pyfile.package().password: - self.core.api.setPackageData(pid, {"password": self.pyfile.package().password}) if self.urls: self.core.api.generateAndAddPackages(self.urls) + def clean(self): + if hasattr(self, "req"): + self.req.close() + del self.req \ No newline at end of file -- cgit v1.2.3 From 35742c2cb023ac49ab3056752d2040cdb030cc2b Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 1 Jan 2012 13:36:59 +0100 Subject: Happy new Year ! --- module/plugins/Crypter.py | 92 +++++++++++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 34 deletions(-) (limited to 'module/plugins/Crypter.py') diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index fc54b32d7..7c76afee7 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- +from traceback import print_exc + from module.Api import Destination from module.common.packagetools import parseNames -from module.utils import to_list -from module.utils.fs import exists +from module.utils import to_list, has_method +from module.utils.fs import exists, remove, fs_encode from Base import Base, Retry @@ -17,6 +19,12 @@ class Package: def addUrl(self, url): self.urls.append(url) + def __eq__(self, other): + return self.name == other.name and self.urls == other.urls + + def __repr__(self): + return " Date: Sun, 1 Jan 2012 18:01:25 +0100 Subject: fixed imports --- module/plugins/Crypter.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'module/plugins/Crypter.py') diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index 7c76afee7..c645f2a72 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -159,8 +159,8 @@ class Crypter(Base): self.decrypt() result = self.convertPackages() else: - if not has_method(cls, "decryptFile"): - self.logDebug("No Decrypting method was overwritten in plugin %s" % self.__name__) + if not has_method(cls, "decryptFile") or urls: + self.logDebug("No suited decrypting method was overwritten in plugin") result = [] if has_method(cls, "decryptFile"): @@ -199,7 +199,7 @@ class Crypter(Base): for url in urls: path = None if url.startswith("http"): - path = None # skip directly + pass elif exists(url): path = url elif exists(self.core.path(url)): @@ -212,7 +212,6 @@ class Crypter(Base): f.close() except IOError, e: self.logError("IOError", e) - remote.append(url) else: remote.append(url) -- cgit v1.2.3 From 4a3a81b63cd85cc3dcd9669868a2079da65838a2 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 3 Jan 2012 20:41:23 +0100 Subject: fixes for old style decrypter --- module/plugins/Crypter.py | 61 +++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 21 deletions(-) (limited to 'module/plugins/Crypter.py') diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index c645f2a72..fe7f0deb8 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -27,23 +27,27 @@ class Package: class PyFileMockup: """ Legacy class needed by old crypter plugins """ - def __init__(self, url): + def __init__(self, url, pack): self.url = url self.name = url + self._package = pack + self.packageid = pack.id if pack else -1 + + def package(self): + return self._package class Crypter(Base): """ Base class for (de)crypter plugins. Overwrite decrypt* methods. - How to use decrypt* methods - --------------------------- + How to use decrypt* methods: You have to overwrite at least one method of decryptURL, decryptURLs, decryptFile. - After decrypting and generating urls/packages you have to return the result at the\ - end of your method. Valid return Data is: + After decrypting and generating urls/packages you have to return the result. + Valid return Data is: - `Package` instance + :class:`Package` instance Crypter.Package A **new** package will be created with the name and the urls of the object. List of urls and `Package` instances @@ -52,9 +56,13 @@ class Crypter(Base): """ + #: Prefix to annotate that the submited string for decrypting is indeed file content + CONTENT_PREFIX = "filecontent:" + @classmethod def decrypt(cls, core, url_or_urls): """Static method to decrypt, something. Can be used by other plugins. + To decrypt file content prefix the string with ``CONTENT_PREFIX `` as seen above. :param core: pyLoad `Core`, needed in decrypt context :param url_or_urls: List of urls or single url/ file content @@ -105,15 +113,18 @@ class Crypter(Base): """Decrypt a single url :param url: url to decrypt - :return: See `Crypter` Documentation + :return: See :class:`Crypter` Documentation """ - raise NotImplementedError + if url.startswith("http"): # basic method to redirect + return self.decryptFile(self.load(url)) + else: + self.fail(_("Not existing file or unsupported protocol")) def decryptURLs(self, urls): """Decrypt a bunch of urls :param urls: list of urls - :return: See `Crypter` Documentation + :return: See :class:`Crypter` Documentation """ raise NotImplementedError @@ -121,12 +132,12 @@ class Crypter(Base): """Decrypt file content :param content: content to decrypt as string - :return: See `Crypter Documentation + :return: See :class:`Crypter` Documentation """ raise NotImplementedError def generatePackages(self, urls): - """Generates `Package` instances and names from urls. Usefull for many different link and no\ + """Generates :class:`Package` instances and names from urls. Usefull for many different links and no\ given package name. :param urls: list of urls @@ -155,9 +166,12 @@ class Crypter(Base): result.extend(to_list(self.decryptURL(url))) elif has_method(cls, "decrypt"): self.logDebug("Deprecated .decrypt() method in Crypter plugin") - self.setup() - self.decrypt() - result = self.convertPackages() + result = [] + for url in urls: + self.pyfile = PyFileMockup(url, self.package) + self.setup() + self.decrypt(self.pyfile) + result.extend(self.convertPackages()) else: if not has_method(cls, "decryptFile") or urls: self.logDebug("No suited decrypting method was overwritten in plugin") @@ -175,7 +189,7 @@ class Crypter(Base): return result def processDecrypt(self, urls): - """ Catches all exceptions in decrypt methods and return results + """Catches all exceptions in decrypt methods and return results :return: Decrypting results """ @@ -187,10 +201,10 @@ class Crypter(Base): return [] def getLocalContent(self, urls): - """Load files from disk + """Load files from disk and seperate to file content and url list :param urls: - :return: content, remote urls + :return: list of (filename, content), remote urls """ content = [] # do nothing if no decryptFile method @@ -198,8 +212,10 @@ class Crypter(Base): remote = [] for url in urls: path = None - if url.startswith("http"): + if url.startswith("http"): # skip urls directly pass + elif url.startswith(self.CONTENT_PREFIX): + path = url elif exists(url): path = url elif exists(self.core.path(url)): @@ -207,9 +223,12 @@ class Crypter(Base): if path: try: - f = open(fs_encode(path), "rb") - content.append((f.name, f.read())) - f.close() + if path.startswith(self.CONTENT_PREFIX): + content.append(("", path[len(self.CONTENT_PREFIX)])) + else: + f = open(fs_encode(path), "rb") + content.append((f.name, f.read())) + f.close() except IOError, e: self.logError("IOError", e) else: -- cgit v1.2.3 From b877847094b0ba03a098dff0fd769eb456b48dd1 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Fri, 6 Jan 2012 17:54:53 +0100 Subject: several improvements, also closes #486, #487 --- module/plugins/Crypter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/plugins/Crypter.py') diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index fe7f0deb8..5d164da64 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -4,7 +4,7 @@ from traceback import print_exc from module.Api import Destination from module.common.packagetools import parseNames -from module.utils import to_list, has_method +from module.utils import to_list, has_method, uniqify from module.utils.fs import exists, remove, fs_encode from Base import Base, Retry @@ -83,7 +83,7 @@ class Crypter(Base): else: # single url ret.append(url_or_pack) # eliminate duplicates - return set(ret) + return uniqify(ret) def __init__(self, core, package=None, password=None): Base.__init__(self, core) -- cgit v1.2.3 From 1bb6ebf544b43cacf7c0755c5a8608b79b95e2d6 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 7 Jan 2012 20:11:16 +0100 Subject: MultiHoster plugin type, some fixes, new documentation structure --- module/plugins/Crypter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'module/plugins/Crypter.py') diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index 5d164da64..3e423881e 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -23,7 +23,10 @@ class Package: return self.name == other.name and self.urls == other.urls def __repr__(self): - return " Date: Sat, 7 Jan 2012 23:15:49 +0100 Subject: xfilesharengprofolder using new crypter api, allow slash in package name --- module/plugins/Crypter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'module/plugins/Crypter.py') diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index 3e423881e..6079ae8f6 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -12,7 +12,7 @@ from Base import Base, Retry class Package: """ Container that indicates new package should be created """ def __init__(self, name, urls=None, dest=Destination.Queue): - self.name = name, + self.name = name self.urls = urls if urls else [] self.dest = dest -- cgit v1.2.3 From 4df2b77fdf42046fe19bd371be7c7255986b5980 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 6 Mar 2012 13:36:39 +0100 Subject: renamed hooks to addons, new filemanager and database, many new api methods you will loose ALL your LINKS, webinterface will NOT work --- module/plugins/Crypter.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) (limited to 'module/plugins/Crypter.py') diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index 6079ae8f6..15feea8e0 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -2,7 +2,6 @@ from traceback import print_exc -from module.Api import Destination from module.common.packagetools import parseNames from module.utils import to_list, has_method, uniqify from module.utils.fs import exists, remove, fs_encode @@ -11,22 +10,33 @@ from Base import Base, Retry class Package: """ Container that indicates new package should be created """ - def __init__(self, name, urls=None, dest=Destination.Queue): + def __init__(self, name, urls=None): self.name = name self.urls = urls if urls else [] - self.dest = dest + # nested packages + self.packs = [] - def addUrl(self, url): + def addURL(self, url): self.urls.append(url) + def addPackage(self, pack): + self.packs.append(pack) + + def getAllURLs(self): + urls = self.urls + for p in self.packs: + urls.extend(p.getAllURLs()) + return urls + + # same name and urls is enough to be equal for packages def __eq__(self, other): return self.name == other.name and self.urls == other.urls def __repr__(self): - return u" Date: Sun, 22 Apr 2012 19:56:17 +0200 Subject: a huge pile of spelling fixes --- module/plugins/Crypter.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'module/plugins/Crypter.py') diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index 15feea8e0..920009f44 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -9,7 +9,7 @@ from module.utils.fs import exists, remove, fs_encode from Base import Base, Retry class Package: - """ Container that indicates new package should be created """ + """ Container that indicates that a new package should be created """ def __init__(self, name, urls=None): self.name = name self.urls = urls if urls else [] @@ -102,14 +102,14 @@ class Crypter(Base): Base.__init__(self, core) self.req = core.requestFactory.getRequest(self.__name__) - # Package the plugin was initialized for, dont use this, its not guaranteed to be set + # Package the plugin was initialized for, don't use this, its not guaranteed to be set self.package = package #: Password supplied by user self.password = password #: Propose a renaming of the owner package self.rename = None - # For old style decrypter, do not use these ! + # For old style decrypter, do not use these! self.packages = [] self.urls = [] self.pyfile = None @@ -120,7 +120,7 @@ class Crypter(Base): """More init stuff if needed""" def setup(self): - """Called everytime before decrypting. A Crypter plugin will be most likly used for several jobs.""" + """Called everytime before decrypting. A Crypter plugin will be most likely used for several jobs.""" def decryptURL(self, url): """Decrypt a single url @@ -150,7 +150,7 @@ class Crypter(Base): raise NotImplementedError def generatePackages(self, urls): - """Generates :class:`Package` instances and names from urls. Usefull for many different links and no\ + """Generates :class:`Package` instances and names from urls. Useful for many different links and no\ given package name. :param urls: list of urls @@ -166,7 +166,7 @@ class Crypter(Base): """ cls = self.__class__ - # seperate local and remote files + # separate local and remote files content, urls = self.getLocalContent(urls) if has_method(cls, "decryptURLs"): @@ -214,7 +214,7 @@ class Crypter(Base): return [] def getLocalContent(self, urls): - """Load files from disk and seperate to file content and url list + """Load files from disk and separate to file content and url list :param urls: :return: list of (filename, content), remote urls -- cgit v1.2.3