From 1bb6ebf544b43cacf7c0755c5a8608b79b95e2d6 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 7 Jan 2012 20:11:16 +0100 Subject: MultiHoster plugin type, some fixes, new documentation structure --- docs/plugins/account_plugin.rst | 5 ++ docs/plugins/base_plugin.rst | 5 ++ docs/plugins/crypter_plugin.rst | 5 ++ docs/plugins/hook_plugin.rst | 162 ++++++++++++++++++++++++++++++++++++++++ docs/plugins/hoster_plugin.rst | 102 +++++++++++++++++++++++++ docs/plugins/overview.rst | 31 ++++++++ 6 files changed, 310 insertions(+) create mode 100644 docs/plugins/account_plugin.rst create mode 100644 docs/plugins/base_plugin.rst create mode 100644 docs/plugins/crypter_plugin.rst create mode 100644 docs/plugins/hook_plugin.rst create mode 100644 docs/plugins/hoster_plugin.rst create mode 100755 docs/plugins/overview.rst (limited to 'docs/plugins') diff --git a/docs/plugins/account_plugin.rst b/docs/plugins/account_plugin.rst new file mode 100644 index 000000000..75bf61a75 --- /dev/null +++ b/docs/plugins/account_plugin.rst @@ -0,0 +1,5 @@ +.. _account_plugin: + +Account - Premium Access +======================== + diff --git a/docs/plugins/base_plugin.rst b/docs/plugins/base_plugin.rst new file mode 100644 index 000000000..62ab248ef --- /dev/null +++ b/docs/plugins/base_plugin.rst @@ -0,0 +1,5 @@ +.. _base_plugin: + +Base Plugin - And here it begins... +=================================== + diff --git a/docs/plugins/crypter_plugin.rst b/docs/plugins/crypter_plugin.rst new file mode 100644 index 000000000..d910ec412 --- /dev/null +++ b/docs/plugins/crypter_plugin.rst @@ -0,0 +1,5 @@ +.. _crypter_plugin: + +Crypter - Extract links from pages +================================== + diff --git a/docs/plugins/hook_plugin.rst b/docs/plugins/hook_plugin.rst new file mode 100644 index 000000000..e263ece2e --- /dev/null +++ b/docs/plugins/hook_plugin.rst @@ -0,0 +1,162 @@ +.. _write_hooks: + +Hook - Do everything you want +============================= + +A Hook is a python file which is located at :file:`module/plugins/hooks`. +The :class:`HookManager ` will load it automatically on startup. Only one instance exists +over the complete lifetime of pyload. Your hook can interact on various events called by the :class:`HookManager `, +do something complete autonomic and has full access to the :class:`Api ` and every detail of pyLoad. +The UpdateManager, CaptchaTrader, UnRar and many more are realised as hooks. + +Hook header +----------- + +Your hook needs to subclass :class:`Hook ` and will inherit all of its method, make sure to check its documentation! + +All Hooks should start with something like this: :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + __name__ = "YourHook" + __version__ = "0.1" + __description__ = "Does really cool stuff" + __config__ = [ ("activated" , "bool" , "Activated" , "True" ) ] + __threaded__ = ["downloadFinished"] + __author_name__ = ("Me") + __author_mail__ = ("me@has-no-mail.com") + +All meta-data is defined in the header, you need at least one option at ``__config__`` so the user can toggle your +hook on and off. Dont't overwrite the ``init`` method if not neccesary, use ``setup`` instead. + +Using the Config +---------------- + +We are taking a closer look at the ``__config__`` parameter. +You can add more config values as desired by adding tuples of the following format to the config list: ``("name", "type", "description", "default value")``. +When everything went right you can access the config values with ``self.getConfig(name)`` and ``self.setConfig(name,value``. + + +Interacting on Events +--------------------- + +The next step is to think about where your Hook action takes places. + +The easiest way is to overwrite specific methods defined by the :class:`Hook ` base class. +The name is indicating when the function gets called. +See :class:`Hook ` page for a complete listing. + +You should be aware of the arguments the Hooks are called with, whether its a :class:`PyFile ` +or :class:`PyPackage ` you should read its related documentation to know how to access her great power and manipulate them. + +A basic excerpt would look like: :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + + def coreReady(self): + print "Yay, the core is ready let's do some work." + + def downloadFinished(self, pyfile): + print "A Download just finished." + +Another important feature to mention can be seen at the ``__threaded__`` parameter. Function names listed will be executed +in a thread, in order to not block the main thread. This should be used for all kind of longer processing tasks. + +Another and more flexible and powerful way is to use event listener. +All hook methods exists as event and very useful additional events are dispatched by the core. For a little overview look +at :class:`HookManager `. Keep in mind that you can define own events and other people may listen on them. + +For your convenience it's possible to register listeners automatical via the ``event_map`` attribute. +It requires a `dict` that maps event names to function names or a `list` of function names. It's important that all names are strings :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + event_map = {"downloadFinished" : "doSomeWork", + "allDownloadsFnished": "someMethod", + "coreReady": "initialize"} + + def initialize(self): + print "Initialized." + + def doSomeWork(self, pyfile): + print "This is equivalent to the above example." + + def someMethod(self): + print "The underlying event (allDownloadsFinished) for this method is not available through the base class" + +An advantage of the event listener is that you are able to register and remove the listeners at runtime. +Use `self.manager.addEvent("name", function)`, `self.manager.removeEvent("name", function)` and see doc for +:class:`HookManager `. Contrary to ``event_map``, ``function`` has to be a reference +and **not** a `string`. + +We introduced events because it scales better if there a a huge amount of events and hooks. So all future interaction will be exclusive +available as event and not accessible through overwriting hook methods. However you can safely do this, it will not be removed and is easier to implement. + + +Providing RPC services +---------------------- + +You may noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. +So probably clients want to be able to interact with your hook to request it's state or invoke some action. + +Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: + + from module.plugins.Hook import Hook, Expose + + class YourHook(Hook): + """ + Your Hook code here. + """ + + @Expose + def invoke(self, arg): + print "Invoked with", arg + +Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. +Here is a basic example: :: + + #Assuming client is a ThriftClient or Api object + + print client.getServices() + print client.call(ServiceCall("YourHook", "invoke", "an argument")) + +Providing status information +---------------------------- +Your hook can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. + +Just store everything in ``self.info``. :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + + def setup(self): + self.info = {"running": False} + + def coreReady(self): + self.info["running"] = True + +Usable with: :: + + #Assuming client is a ThriftClient or Api object + + print client.getAllInfo() + +Example +------- + Sorry but you won't find an example here ;-) + + Look at :file:`module/plugins/hooks` and you will find plenty examples there. diff --git a/docs/plugins/hoster_plugin.rst b/docs/plugins/hoster_plugin.rst new file mode 100644 index 000000000..59f35f5cb --- /dev/null +++ b/docs/plugins/hoster_plugin.rst @@ -0,0 +1,102 @@ +.. _hoster_plugin: + +Hoster - Load files to disk +=========================== + +A Plugin is a python file located at one of the subfolders in :file:`module/plugins/`. Either :file:`hoster`, :file:`crypter` +or :file:`container`, depending of it's type. + +There are three kinds of different plugins: **Hoster**, **Crypter**, **Container**. +All kind of plugins inherit from the base :class:`Plugin `. You should know its +convenient methods, they make your work easier ;-) + +Every plugin defines a ``__pattern__`` and when the user adds urls, every url is matched against the pattern defined in +the plugin. In case the ``__pattern__`` matched on the url the plugin will be assigned to handle it and instanciated when +pyLoad begins to download/decrypt the url. + +Plugin header +------------- + +How basic hoster plugin header could look like: :: + + from module.plugin.Hoster import Hoster + + class MyFileHoster(Hoster): + __version__ = "0.1" + __pattern__ = r"http://myfilehoster.example.com/file_id/[0-9]+" + __config__ = [] + +You have to define these meta-data, ``__pattern__`` has to be a regexp that sucessfully compiles with +``re.compile(__pattern__)``. + +Just like :ref:`write_hooks` you can add and use config values exatly the same way. +If you want a Crypter or Container plugin, just replace the word Hoster with your desired plugin type. + + +Hoster plugins +-------------- + +We head to the next important section, the ``process`` method of your plugin. +In fact the ``process`` method is the only functionality your plugin has to provide, but its always a good idea to split up tasks to not produce spaghetti code. +An example ``process`` function could look like this :: + + from module.plugin.Hoster import Hoster + + class MyFileHoster(Hoster): + """ + plugin code + """ + + def process(self, pyfile): + html = self.load(pyfile.url) # load the content of the orginal pyfile.url to html + + # parse the name from the site and set attribute in pyfile + pyfile.name = self.myFunctionToParseTheName(html) + parsed_url = self.myFunctionToParseUrl(html) + + # download the file, destination is determined by pyLoad + self.download(parsed_url) + +You need to know about the :class:`PyFile ` class, since an instance of it is given as parameter to every pyfile. +Some tasks your plugin should handle: proof if file is online, get filename, wait if needed, download the file, etc.. + +Wait times +__________ + +Some hoster require you to wait a specific time. Just set the time with ``self.setWait(seconds)`` or +``self.setWait(seconds, True)`` if you want pyLoad to perform a reconnect if needed. + +Captcha decrypting +__________________ + +To handle captcha input just use ``self.decryptCaptcha(url, ...)``, it will be send to clients +or handled by :class:`Hook ` plugins + +Crypter +------- + +What about Decrypter and Container plugins? +Well, they work nearly the same, only that the function they have to provide is named ``decrypt`` + +Example: :: + + from module.plugin.Crypter import Crypter + + class MyFileCrypter(Crypter): + """ + plugin code + """ + def decrypt(self, pyfile): + + urls = ["http://get.pyload.org/src", "http://get.pyload.org/debian", "http://get.pyload.org/win"] + + self.packages.append(("pyLoad packages", urls, "pyLoad packages")) # urls list of urls + +They can access all the methods from :class:`Plugin `, but the important thing is they +have to append all packages they parsed to the `self.packages` list. Simply append tuples with `(name, urls, folder)`, +where urls is the list of urls contained in the packages. Thats all of your work, pyLoad will know what to do with them. + +Examples +-------- + +Best examples are already existing plugins in :file:`module/plugins/`. \ No newline at end of file diff --git a/docs/plugins/overview.rst b/docs/plugins/overview.rst new file mode 100755 index 000000000..23913b787 --- /dev/null +++ b/docs/plugins/overview.rst @@ -0,0 +1,31 @@ +.. _overview: + +================ +Extending pyLoad +================ + +.. pull-quote:: + Any sufficiently advanced technology is indistinguishable from magic. + + -- Arthur C. Clarke + + +.. rubric:: Motivation + +pyLoad offers an comfortable and powerful plugin system to make extending possible. With it you only need to have some +python knowledge and can just start right away writing your own plugins. This document gives you an overwiew about the +conceptual part. You should not left out the `Base` part, since it contains basic functionality for all plugins types. + +.. rubric:: Contents + +.. toctree:: + + base_plugin.rst + crypter_plugin.rst + hoster_plugin.rst + account_plugin.rst + hook_plugin.rst + + + +.. rubric:: Footnotes \ No newline at end of file -- cgit v1.2.3