diff options
| -rw-r--r-- | docs/access_api.rst | 3 | ||||
| -rwxr-xr-x | docs/extend_pyload.rst | 13 | ||||
| -rw-r--r-- | docs/index.rst | 2 | ||||
| -rw-r--r-- | docs/module_overview.rst | 4 | ||||
| -rw-r--r-- | docs/write_hooks.rst | 162 | ||||
| -rw-r--r--[-rwxr-xr-x] | docs/write_plugins.rst | 133 | ||||
| -rw-r--r-- | module/Api.py | 2 | ||||
| -rw-r--r-- | module/HookManager.py | 8 | ||||
| -rw-r--r-- | module/Progress.py | 5 | ||||
| -rw-r--r-- | module/database/FileDatabase.py | 8 | ||||
| -rw-r--r-- | module/plugins/hooks/UnRar.py | 63 |
11 files changed, 321 insertions, 82 deletions
diff --git a/docs/access_api.rst b/docs/access_api.rst index dd3998df4..d70f120c3 100644 --- a/docs/access_api.rst +++ b/docs/access_api.rst @@ -43,7 +43,8 @@ at the thrift wiki and the examples here http://wiki.apache.org/thrift/ThriftUsa Example ------- In case you want to use python, pyload has already all files included to access the api over rpc. -A basic script that prints out some information. :: + +A basic script that prints out some information: :: from module.remote.thriftbackend.ThriftClient import ThriftClient, WrongLogin diff --git a/docs/extend_pyload.rst b/docs/extend_pyload.rst new file mode 100755 index 000000000..337cb6854 --- /dev/null +++ b/docs/extend_pyload.rst @@ -0,0 +1,13 @@ +.. _extend_pyload: + +******************** +How to extend pyLoad +******************** + +In general there a two different plugin types. These allow everybody to write powerful, modular plugins without knowing +every detail of the pyLoad core. However you should have some basic knowledge of python. + +.. toctree:: + + write_hooks.rst + write_plugins.rst
\ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index ce49325e7..757fd7537 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,7 +17,7 @@ Contents: :maxdepth: 2 access_api.rst - write_plugins.rst + extend_pyload.rst module_overview.rst .. currentmodule:: module diff --git a/docs/module_overview.rst b/docs/module_overview.rst index b7ca5e05b..a3c31a88b 100644 --- a/docs/module_overview.rst +++ b/docs/module_overview.rst @@ -1,8 +1,7 @@ Module Overview =============== -You can find an overview of important classes here: - +You can find an overview of some important classes here: .. autosummary:: :toctree: module @@ -12,5 +11,6 @@ You can find an overview of important classes here: module.plugins.Crypter.Crypter module.plugins.Account.Account module.plugins.Hook.Hook + module.HookManager.HookManager module.PyFile.PyFile module.PyPackage.PyPackage diff --git a/docs/write_hooks.rst b/docs/write_hooks.rst new file mode 100644 index 000000000..ffc41d705 --- /dev/null +++ b/docs/write_hooks.rst @@ -0,0 +1,162 @@ +.. _write_hooks: + +Hooks +===== + +A Hook is a python file which is located at :file:`module/plugins/hooks`. +The :class:`HookManager <module.HookManager.HookManager>` will load it automatically on startup. Only one instance exists +over the complete lifetime of pyload. Your hook can interact on various events called by the :class:`HookManager <module.HookManager.HookManager>`, +do something complete autonomic and has full access to the :class:`Api <module.Api.Api>` and every detail of pyLoad. +The UpdateManager, CaptchaTrader, UnRar and many more are realised as hooks. + +Hook header +----------- + +Your hook needs to subclass :class:`Hook <module.plugins.Hook.Hook>` and will inherit all of its method, make sure to check its documentation! + +All Hooks should start with something like this: :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + __name__ = "My own Hook" + __version__ = "0.1" + __description__ = "Does really cool stuff" + __config__ = [ ("activated" , "bool" , "Activated" , "True" ) ] + __threaded__ = ["downloadFinished"] + __author_name__ = ("Me") + __author_mail__ = ("me@has-no-mail.com") + +All meta-data is defined in the header, you need at least one option at ``__config__`` so the user can toggle your +hook on and off. Dont't overwrite the ``init`` method if not neccesary, use ``setup`` instead. + +Using the Config +---------------- + +We are taking a closer look at the ``__config__`` parameter. +You can add more config values as desired by adding tuples of the following format to the config list: ``("name", "type", "description", "default value")``. +When everything went right you can access the config values with ``self.getConfig(name)`` and ``self.setConfig(name,value``. + + +Interacting on Events +--------------------- + +The next step is to think about where your Hook action takes places. + +The easiest way is to overwrite specific methods defined by the :class:`Hook <module.plugins.Hook.Hook>` base class. +The name is indicating when the function gets called. +See :class:`Hook <module.plugins.Hook.Hook>` page for a complete listing. + +You should be aware of the arguments the Hooks are called with, whether its a :class:`PyFile <module.PyFile.PyFile>` +or :class:`PyPackage <module.PyPackage.PyPackage>` you should read its related documentation to know how to access her great power and manipulate them. + +A basic excerpt would look like: :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + + def coreReady(self): + print "Yay, the core is ready let's do some work." + + def downloadFinished(self, pyfile): + print "A Download just finished." + +Another important feature to mention can be seen at the ``__threaded__`` parameter. Function names listed will be executed +in a thread, in order to not block the main thread. This should be used for all kind of longer processing tasks. + +Another and more flexible and powerful way is to use event listener. +All hook methods exists as event and very useful additional events are dispatched by the core. For a little overview look +at :class:`HookManager <module.HookManager.HookManager>`. Keep in mind that you can define own events and other people may listen on them. + +For your convenience it's possible to register listeners automatical via the ``event_map`` attribute. +It requires a `dict` that maps event names to function names or a `list` of function names. It's important that all names are strings :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + event_map = {"downloadFinished" : "doSomeWork", + "allDownloadsFnished": "someMethod", + "coreReady": "initialize"} + + def initialize(self): + print "Initialized." + + def doSomeWork(self, pyfile): + print "This is equivalent to the above example." + + def someMethod(self): + print "The underlying event (allDownloadsFinished) for this method is not available through the base class" + +An advantage of the event listener is that you are able to register and remove the listeners at runtime. +Use `self.manager.addEvent("name", function)`, `self.manager.removeEvent("name", function)` and see doc for +:class:`HookManager <module.HookManager.HookManager>`. Contrary to ``event_map``, ``function`` has to be a reference +and **not** a `string`. + +We introduced events because it scales better if there a a huge amount of events and hooks. So all future interaction will be exclusive +available as event and not accessible through overwriting hook methods. However you can safely do this, it will not be removed and is easier to implement. + + +Providing RPC services +---------------------- + +You may noticed that pyLoad has an :class:`Api <module.Api.Api>`, which can be used internal or called by clients via RPC. +So probably clients want to be able to interact with your hook to request it's state or invoke some action. + +Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: + + from module.plugins.Hook import Hook, Expose + + class YourHook(Hook): + """ + Your Hook code here. + """ + + @Expose + def invoke(self, arg): + print "Invoked with", arg + +Thats all, it's available via the :class:`Api <module.Api.Api>` now. If you want to use it read :ref:`access_api`. +Here is a basic example: :: + + #Assuming client is a ThriftClient or Api object + + print client.getServices() + print client.call(ServiceCall("YourHook", "invoke", "an argument")) + +Providing status information +---------------------------- +Your hook can store information in a ``dict`` that can easily be retrievied via the :class:`Api <module.Api.Api>`. + +Just store everything in ``self.info``. :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + + def setup(self): + self.info = {"running": False} + + def coreReady(self): + self.info["running"] = True + +Usable with: :: + + #Assuming client is a ThriftClient or Api object + + print client.getAllInfo() + +Example +------- + Sorry but you won't find an example here ;-) + + Look at :file:`module/plugins/hooks` and you will find plenty examples there.
\ No newline at end of file diff --git a/docs/write_plugins.rst b/docs/write_plugins.rst index 0c581cdab..b513a5978 100755..100644 --- a/docs/write_plugins.rst +++ b/docs/write_plugins.rst @@ -1,92 +1,103 @@ -.. _write_plugins: +.. _write_plugins: -******************** -How to extend pyLoad -******************** - -This page should give you an basic idea, how to extend pyLoad with new features! - -Hooks ------ - -A Hook is a module which is located at :file:`module/plugin/hooks` and loaded with the startup. It inherits all methods from the base :class:`Hook <module.plugins.Hook.Hook>` module, make sure to check its documentation! - -All Hooks should start with something like this: :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - __name__ = "My own Hook" - __version__ = "0.1" - __description__ = "Does really cool stuff" - __config__ = [ ("activated" , "bool" , "Activated" , "True" ) ] - __threaded__ = ["downloadFinished"] - __author_name__ = ("Me") - __author_mail__ = ("me@has-no-mail.com") - - def setup(self): - #init hook and other stuff here - pass - -Take a closer look at the ``__config__`` parameter. You need at least this option indicating whether your Hook is activated or not. -You can add more config values as desired by adding tuples of the following format to the config list: ``("name", "type", "description", "default value")``. -When everything went right you can access your config values with ``self.getConfig(name)``. - -The next step is to think about where your Hook action takes places. -You can overwrite several methods of the base plugin, her name indicates when they gets called. -See :class:`Hook <module.plugins.Hook.Hook>` page for a complete listing. - -You should be aware of the arguments the Hooks are called with, whether its a :class:`PyFile <module.FileDatabase.PyFile>` or :class:`PyPackage <module.FileDatabase.PyPackage>` you should read its related documentation to know how to access her great power and manipulate them. - -Another important feature to mention can be seen at the ``__threaded__`` parameter. Function names listed will be executed in a thread, in order to not block the main thread. This should be used for all kind of longer processing tasks. - - Plugins -------- +======= -A Plugin in pyLoad sense is a module, that will be loaded and executed when its pattern match to a url that was been added to pyLoad. +A Plugin is a python file located at one of the subfolders in :file:`module/plugins/`. Either :file:`hoster`, :file:`crypter` +or :file:`container`, depending of it's type. There are three kinds of different plugins: **Hoster**, **Crypter**, **Container**. -All kind of plugins inherit from the base :class:`Plugin <module.plugins.Plugin.Plugin>`. You should know its convenient methods, they make your work easier ;-) +All kind of plugins inherit from the base :class:`Plugin <module.plugins.Plugin.Plugin>`. You should know its +convenient methods, they make your work easier ;-) -We take a look how basis hoster plugin header could look like: :: +Every plugin defines a ``__pattern__`` and when the user adds urls, every url is matched against the pattern defined in +the plugin. In case the ``__pattern__`` matched on the url the plugin will be assigned to handle it and instanciated when +pyLoad begins to download/decrypt the url. + +Plugin header +------------- + +How basic hoster plugin header could look like: :: from module.plugin.Hoster import Hoster - + class MyFileHoster(Hoster): __name__ = "MyFileHoster" __version__ = "0.1" __pattern__ = r"http://myfilehoster.example.com/file_id/[0-9]+" __config__ = [] -Thats it, like above with :ref:`Hooks` you can add config values exatly the same way. +You have to define these meta-data, ``__pattern__`` has to be a regexp that sucessfully compiles with +``re.compile(__pattern__)``. -The ``__pattern__`` property is very important, it will be checked against every added url and if it matches your plugin will be choosed to do the grateful task of processing this download! In case you dont already spotted it, ``__pattern__`` has to be a regexp of course. +Just like :ref:`write_hooks` you can add and use config values exatly the same way. +If you want a Crypter or Container plugin, just replace the word Hoster with your desired plugin type. + + +Hoster plugins +-------------- We head to the next important section, the ``process`` method of your plugin. In fact the ``process`` method is the only functionality your plugin has to provide, but its always a good idea to split up tasks to not produce spaghetti code. -An example ``process`` function could look like this :: +An example ``process`` function could look like this :: - def process(self, pyfile): - html = self.load(pyfile.url) #load the content of the orginal pyfile.url to html + from module.plugin.Hoster import Hoster - pyfile.name = self.myFunctionToParseTheName(html) #parse the name from the site and write it to pyfile + class MyFileHoster(Hoster): + """ + plugin code + """ + + def process(self, pyfile): + html = self.load(pyfile.url) # load the content of the orginal pyfile.url to html + + # parse the name from the site and set attribute in pyfile + pyfile.name = self.myFunctionToParseTheName(html) parsed_url = self.myFunctionToParseUrl(html) - self.download(parsed_url) # download the file + # download the file, destination is determined by pyLoad + self.download(parsed_url) -You need to know about the :class:`PyFile <module.FileDatabase.PyFile>` class, since a instance of it is given as parameter to every pyfile. +You need to know about the :class:`PyFile <module.PyFile.PyFile>` class, since an instance of it is given as parameter to every pyfile. Some tasks your plugin should handle: proof if file is online, get filename, wait if needed, download the file, etc.. -There are also some functions to mention which are provided by the base Plugin: ``self.wait()``, ``self.decryptCaptcha()`` -Read about there functionality in the :class:`Plugin <module.plugins.Plugin.Plugin>` doc. +Wait times +__________ + +Some hoster require you to wait a specific time. Just set the time with ``self.setWait(seconds)`` or +``self.setWait(seconds, True)`` if you want pyLoad to perform a reconnect if needed. + +Captcha decrypting +__________________ + +To handle captcha input just use ``self.decryptCaptcha(url, ...)``, it will be send to clients +or handled by :class:`Hook <module.plugins.Hook.Hook>` plugins + +Crypter +------- What about Decrypter and Container plugins? Well, they work nearly the same, only that the function they have to provide is named ``decrypt`` -Example: :: +Example: :: + + from module.plugin.Crypter import Crypter + + class MyFileCrypter(Crypter): + """ + plugin code + """ + def decrypt(self, pyfile): + + urls = ["http://get.pyload.org/src", "http://get.pyload.org/debian", "http://get.pyload.org/win"] + + self.packages.append(("pyLoad packages", urls, "pyLoad packages")) # urls list of urls + +They can access all the methods from :class:`Plugin <module.plugins.Plugin.Plugin>`, but the important thing is they +have to append all packages they parsed to the `self.packages` list. Simply append tuples with `(name, urls, folder)`, +where urls is the list of urls contained in the packages. Thats all of your work, pyLoad will know what to do with them. - def decrypt(self, pyfile): - self.packages.append((name, urls, folder)) # urls list of urls +Examples +-------- -They can access all the methods from :class:`Plugin <module.plugins.Plugin.Plugin>`, but the important thing is they have to append all packages they parsed to the `self.packages` list. Simply append tuples with `(name, urls, folder)`, where urls is the list of urls contained in the packages. Thats all of your work, pyLoad will know what to do with them. +Best examples are already existing plugins in :file:`module/plugins/`.
\ No newline at end of file diff --git a/module/Api.py b/module/Api.py index 15052e83a..975b43709 100644 --- a/module/Api.py +++ b/module/Api.py @@ -152,9 +152,11 @@ class Api(Iface): return serverStatus def freeSpace(self): + """Available free space at download directory in bytes""" return freeSpace(self.core.config["general"]["download_folder"]) def getServerVersion(self): + """pyLoad Core version """ return self.core.version def kill(self): diff --git a/module/HookManager.py b/module/HookManager.py index 393db2de6..94faaba6e 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -34,18 +34,18 @@ class HookManager: but some very usefull events are called by the Core. Contrary to overwriting hook methods you can use event listener, which provides additional entry point in the control flow. - Only use very short tasks or use threads. + Only do very short tasks or use threads. *Known Events:* All hook methods exists as events. downloadPreparing: A download was just queued and will be prepared now. - Argument: fid + Argument: fid downloadStarts: A file will immediately starts the download afterwards. - Argument: fid + Argument: fid linksAdded: Someone just added links, you are able to modify the links. - Arguments: links, pid + Arguments: links, pid allDownloadsProcessed: Every link was handled, pyload would idle afterwards. diff --git a/module/Progress.py b/module/Progress.py index e12786da8..1ce4903a4 100644 --- a/module/Progress.py +++ b/module/Progress.py @@ -34,4 +34,7 @@ class Progress: self.notify() def getPercent(self): - return self.value + try: + return int(self.value) + except: + return 0 diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 96b505fe0..3e2f37976 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -95,8 +95,8 @@ class FileHandler: data = self.db.getAllLinks(queue) packs = self.db.getAllPackages(queue) - data.update([(str(x.id), x.toDbDict()[x.id]) for x in self.cache.itervalues()]) - packs.update([(str(x.id), x.toDict()[x.id]) for x in self.packageCache.itervalues() if x.queue == queue]) + data.update([(str(x.id), x.toDbDict()[x.id]) for x in self.cache.values()]) + packs.update([(str(x.id), x.toDict()[x.id]) for x in self.packageCache.values() if x.queue == queue]) for key, value in data.iteritems(): if packs.has_key(str(value["package"])): @@ -351,9 +351,10 @@ class FileHandler: """checks if all files are finished and dispatch event""" if not self.getQueueCount(True): - #hope its not called twice + #hope its not called together with all DownloadsProcessed self.core.hookManager.dispatchEvent("allDownloadsProcessed") self.core.hookManager.dispatchEvent("allDownloadsFinished") + self.core.log.debug("All downloads finished") return True return False @@ -365,6 +366,7 @@ class FileHandler: if not self.db.processcount(1): self.core.hookManager.dispatchEvent("allDownloadsProcessed") + self.core.log.debug("All downloads processed") return True return False diff --git a/module/plugins/hooks/UnRar.py b/module/plugins/hooks/UnRar.py index 751c03987..6dfe2c195 100644 --- a/module/plugins/hooks/UnRar.py +++ b/module/plugins/hooks/UnRar.py @@ -24,13 +24,14 @@ from module.plugins.Hook import Hook from module.lib.pyunrar import Unrar, WrongPasswordError, CommandError, UnknownError, LowRamError from traceback import print_exc -from os.path import exists, join, isabs -from os import remove +from os.path import exists, join, isabs, isdir +from os import remove, makedirs, rmdir, listdir, chown, chmod +from pwd import getpwnam import re class UnRar(Hook): __name__ = "UnRar" - __version__ = "0.1" + __version__ = "0.11" __description__ = """unrar""" __config__ = [ ("activated", "bool", "Activated", False), ("fullpath", "bool", "extract full path", True), @@ -70,7 +71,31 @@ class UnRar(Hook): self.ram = 0 self.ram /= 1024 - + + def setOwner(self,d,uid,gid,mode): + if not exists(d): + self.core.log.debug(_("Directory %s does not exist!") % d) + return + fileList=listdir(d) + for fileEntry in fileList: + fullEntryName=join(d,fileEntry) + if isdir(fullEntryName): + self.setOwner(fullEntryName,uid,gid,mode) + try: + chown(fullEntryName,uid,gid) + chmod(fullEntryName,mode) + except: + self.core.log.debug(_("Chown/Chmod for %s failed") % fullEntryName) + self.core.log.debug(_("Exception: %s") % sys.exc_info()[0]) + continue + try: + chown(d,uid,gid) + chmod(d,mode) + except: + self.core.log.debug(_("Chown/Chmod for %s failed") % d) + self.core.log.debug(_("Exception: %s") % sys.exc_info()[0]) + return + def addPassword(self, pws): if not type(pws) == list: pws = [pws] pws.reverse() @@ -123,14 +148,15 @@ class UnRar(Hook): pyfile.progress.setRange(0, 100) def s(p): pyfile.progress.setValue(p) - + download_folder = self.core.config['general']['download_folder'] - + self.core.log.debug(_("download folder %s") % download_folder) + if self.core.config['general']['folder_per_package']: folder = join(download_folder, pack.folder.decode(sys.getfilesystemencoding())) else: folder = download_folder - + destination = folder if self.getConfig("unrar_destination") and not self.getConfig("unrar_destination").lower() == "none": destination = self.getConfig("unrar_destination") @@ -141,7 +167,12 @@ class UnRar(Hook): destination = join(destination, sub) else: destination = join(folder, destination, sub) - + + self.core.log.debug(_("Destination folder %s") % destination) + if not exists(destination): + self.core.log.info(_("Creating destination folder %s") % destination) + makedirs(destination) + u = Unrar(join(folder, fname), tmpdir=join(folder, "tmp"), ramSize=(self.ram if self.getConfig("ramwarning") else 0), cpu=self.getConfig("renice")) try: success = u.crackPassword(passwords=self.passwords, statusFunction=s, overwrite=True, destination=destination, fullPath=self.getConfig("fullpath")) @@ -175,9 +206,23 @@ class UnRar(Hook): if success: self.core.log.info(_("Unrar of %s ok") % fname) self.removeFiles(pack, fname) + if self.core.config['general']['folder_per_package']: + if self.getConfig("deletearchive"): + self.core.log.debug(_("Deleting package directory %s...") % folder) + rmdir(folder) + self.core.log.debug(_("Package directory %s has been deleted.") % folder) + ownerUser=self.core.config['permission']['user'] + uinfo=getpwnam(ownerUser) + fileMode=int(self.core.config['permission']['file'],8) + self.core.log.debug(_("Setting destination file/directory owner to %s.") % ownerUser) + self.core.log.debug(_("Setting destination file/directory mode to %s.") % fileMode) + self.core.log.debug(_("Uid is %s.") % uinfo.pw_uid) + self.core.log.debug(_("Gid is %s.") % uinfo.pw_gid) + self.setOwner(destination,uinfo.pw_uid,uinfo.pw_gid,fileMode) + self.core.log.debug(_("The owner/rights have been successfully changed.")) self.core.hookManager.unrarFinished(folder, fname) else: - self.core.log.info(_("Unrar of %s failed (wrong password)") % fname) + self.core.log.info(_("Unrar of %s failed (wrong password or bad parts)") % fname) finally: pyfile.progress.setValue(100) pyfile.setStatus("finished") |