From 1bb6ebf544b43cacf7c0755c5a8608b79b95e2d6 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 7 Jan 2012 20:11:16 +0100 Subject: MultiHoster plugin type, some fixes, new documentation structure --- docs/access_api.rst | 121 ---------- docs/api/access_api.rst | 122 +++++++++++ docs/api/datatypes.rst | 352 ++++++++++++++++++++++++++++++ docs/api/overview.rst | 35 +++ docs/conf.py | 4 +- docs/extend_pyload.rst | 13 -- docs/index.rst | 43 +++- docs/module_overview.rst | 6 +- docs/plugins/account_plugin.rst | 5 + docs/plugins/base_plugin.rst | 5 + docs/plugins/crypter_plugin.rst | 5 + docs/plugins/hook_plugin.rst | 162 ++++++++++++++ docs/plugins/hoster_plugin.rst | 102 +++++++++ docs/plugins/overview.rst | 31 +++ docs/write_hooks.rst | 162 -------------- docs/write_plugins.rst | 103 --------- module/HookManager.py | 2 +- module/network/HTTPDownload.py | 6 +- module/network/HTTPRequest.py | 2 +- module/plugins/Account.py | 3 +- module/plugins/Crypter.py | 5 +- module/plugins/Hook.py | 27 +-- module/plugins/MultiHoster.py | 58 +++++ module/plugins/internal/MultiHoster.py | 2 +- module/remote/thriftbackend/pyload.thrift | 5 +- module/threads/DecrypterThread.py | 2 +- pavement.py | 17 +- 27 files changed, 959 insertions(+), 441 deletions(-) delete mode 100644 docs/access_api.rst create mode 100644 docs/api/access_api.rst create mode 100644 docs/api/datatypes.rst create mode 100644 docs/api/overview.rst delete mode 100755 docs/extend_pyload.rst create mode 100644 docs/plugins/account_plugin.rst create mode 100644 docs/plugins/base_plugin.rst create mode 100644 docs/plugins/crypter_plugin.rst create mode 100644 docs/plugins/hook_plugin.rst create mode 100644 docs/plugins/hoster_plugin.rst create mode 100755 docs/plugins/overview.rst delete mode 100644 docs/write_hooks.rst delete mode 100644 docs/write_plugins.rst create mode 100644 module/plugins/MultiHoster.py diff --git a/docs/access_api.rst b/docs/access_api.rst deleted file mode 100644 index df69da8b2..000000000 --- a/docs/access_api.rst +++ /dev/null @@ -1,121 +0,0 @@ -.. _access_api: - -********************* -How to access the API -********************* - -pyLoad has a very powerfull API with can be accessed in several ways. - -Overview --------- - -First of all, you need to know what you can do with our API. It lets you do all common task like -retrieving download status, manage queue, manage accounts, modify config and so on. - -This document is not intended to explain every function in detail, for a complete listing -see :class:`Api `. - -Of course its possible to access the ``core.api`` attribute in plugins and hooks, but much more -interesting is the possibillity to call function from different programs written in many different languages. - -pyLoad uses thrift as backend and provides its :class:`Api ` as service. -More information about thrift can be found here http://wiki.apache.org/thrift/. - - -Using Thrift ------------- - -Every thrift service has to define all data structures and declare every method which should be usable via rpc. -This file is located :file:`module/remote/thriftbackend/pyload.thrift`, its very helpful to inform about -arguments and detailed structure of return types. However it does not contain any information about what the functions does. - -Assuming you want to use the API in any other language than python than check if it is -supported here http://wiki.apache.org/thrift/LibraryFeatures?action=show&redirect=LanguageSupport. - -Now install thrift, for instructions see http://wiki.apache.org/thrift/ThriftInstallation. -If every thing went fine you are ready to generate the method stubs, the command basically looks like this. :: - - $ thrift --gen (language) pyload.thrift - -You find now a directory named :file:`gen-(language)`. For instruction how to use the generated files consider the docs -at the thrift wiki and the examples here http://wiki.apache.org/thrift/ThriftUsage. - - -======= -Example -======= -In case you want to use python, pyload has already all files included to access the api over rpc. - -A basic script that prints out some information: :: - - from module.remote.thriftbackend.ThriftClient import ThriftClient, WrongLogin - - try: - client = ThriftClient(host="127.0.0.1", port=7227, user="User", password="yourpw") - except: - print "Login was wrong" - exit() - - print "Server version:", client.getServerVersion() - print client.statusDownloads() - q = client.getQueue() - for p in q: - data = client.getPackageData(p.pid) - print "Package Name: ", data.name - -That's all for now, pretty easy isn't it? -If you still have open questions come around in irc or post them at our pyload forum. - - -Using HTTP/JSON ---------------- - -Another maybe easier way, which does not require much setup is to access the JSON Api via HTTP. -For this reason the webinterface must be enabled. - -===== -Login -===== - -First you need to authenticate, if you using this within the webinterface and the user is logged the API is also accessible, -since they share the same cookie/session. - -However, if you are building a external client and want to authenticate manually -you have to send your credentials ``username`` and ``password`` as -POST parameter to ``http://pyload-core/api/login``. - -The result will be your session id. If you are using cookies, it will be set and you can use the API now. -In case you dont't have cookies enabled you can pass the session id as ``session`` POST parameter -so pyLoad can authenticate you. - -=============== -Calling Methods -=============== - -In general you can use any method listed at the :class:`Api ` documentation, which is also available to -the thriftbackend. - -Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address -or hostname including the webinterface port. By default on local access this would be `localhost:8000`. - -The return value will be formatted in JSON, complex data types as dictionaries. -As mentionted above for a documentation about the return types look at the thrift specification file :file:`module/remote/thriftbackend/pyload.thrift`. - -================== -Passing parameters -================== - -To pass arguments you have two choices. -Either use positional arguments, eg ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword arguments -supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names in the :class:`Api ` -documentation. - -It is important that *all* arguments are in JSON format. So ``http://pyload-core/api/getFileData/1`` is valid because -1 represents an integer in json format. On the other hand if the method is expecting strings, this would be correct: -``http://pyload-core/api/getUserData/"username"/"password"``. - -Strings are wrapped in double qoutes, because `"username"` represents a string in json format. It's not limited to strings and intergers, -every container type like lists and dicts are possible. You usually don't have to convert them. just use a json encoder before using them -in the HTTP request. - -Please note that the data have to be urlencoded at last. (Most libaries will do that automatically) \ No newline at end of file diff --git a/docs/api/access_api.rst b/docs/api/access_api.rst new file mode 100644 index 000000000..efa1ae3fc --- /dev/null +++ b/docs/api/access_api.rst @@ -0,0 +1,122 @@ +.. _access_api: + +********************* +How to access the API +********************* + +pyLoad has a very powerfull API with can be accessed in several ways. + +Overview +-------- + +First of all, you need to know what you can do with our API. It lets you do all common task like +retrieving download status, manage queue, manage accounts, modify config and so on. + +This document is not intended to explain every function in detail, for a complete listing +see :class:`Api `. + +Of course its possible to access the ``core.api`` attribute in plugins and hooks, but much more +interesting is the possibillity to call function from different programs written in many different languages. + +pyLoad uses thrift as backend and provides its :class:`Api ` as service. +More information about thrift can be found here http://wiki.apache.org/thrift/. + + +Using Thrift +------------ + +Every thrift service has to define all data structures and declare every method which should be usable via rpc. +This file is located at :file:`module/remote/thriftbackend/pyload.thrift`, its very helpful to inform about +arguments and detailed structure of return types. However it does not contain any information about what the functions does. +You can also look at it :doc:`here ` + +Assuming you want to use the API in any other language than python than check if it is +supported here http://wiki.apache.org/thrift/LibraryFeatures?action=show&redirect=LanguageSupport. + +Now install thrift, for instructions see http://wiki.apache.org/thrift/ThriftInstallation. +If every thing went fine you are ready to generate the method stubs, the command basically looks like this. :: + + $ thrift --gen (language) pyload.thrift + +You find now a directory named :file:`gen-(language)`. For instruction how to use the generated files consider the docs +at the thrift wiki and the examples here http://wiki.apache.org/thrift/ThriftUsage. + + +======= +Example +======= +In case you want to use python, pyload has already all files included to access the api over rpc. + +A basic script that prints out some information: :: + + from module.remote.thriftbackend.ThriftClient import ThriftClient, WrongLogin + + try: + client = ThriftClient(host="127.0.0.1", port=7227, user="User", password="yourpw") + except: + print "Login was wrong" + exit() + + print "Server version:", client.getServerVersion() + print client.statusDownloads() + q = client.getQueue() + for p in q: + data = client.getPackageData(p.pid) + print "Package Name: ", data.name + +That's all for now, pretty easy isn't it? +If you still have open questions come around in irc or post them at our pyload forum. + + +Using HTTP/JSON +--------------- + +Another maybe easier way, which does not require much setup is to access the JSON Api via HTTP. +For this reason the webinterface must be enabled. + +===== +Login +===== + +First you need to authenticate, if you using this within the webinterface and the user is logged the API is also accessible, +since they share the same cookie/session. + +However, if you are building a external client and want to authenticate manually +you have to send your credentials ``username`` and ``password`` as +POST parameter to ``http://pyload-core/api/login``. + +The result will be your session id. If you are using cookies, it will be set and you can use the API now. +In case you dont't have cookies enabled you can pass the session id as ``session`` POST parameter +so pyLoad can authenticate you. + +=============== +Calling Methods +=============== + +In general you can use any method listed at the :class:`Api ` documentation, which is also available to +the thriftbackend. + +Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address +or hostname including the webinterface port. By default on local access this would be `localhost:8000`. + +The return value will be formatted in JSON, complex data types as dictionaries. +As mentionted above for a documentation about the return types look at the thrift specification file :file:`module/remote/thriftbackend/pyload.thrift`. + +================== +Passing parameters +================== + +To pass arguments you have two choices. +Either use positional arguments, eg ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword arguments +supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names in the :class:`Api ` +documentation. + +It is important that *all* arguments are in JSON format. So ``http://pyload-core/api/getFileData/1`` is valid because +1 represents an integer in json format. On the other hand if the method is expecting strings, this would be correct: +``http://pyload-core/api/getUserData/"username"/"password"``. + +Strings are wrapped in double qoutes, because `"username"` represents a string in json format. It's not limited to strings and intergers, +every container type like lists and dicts are possible. You usually don't have to convert them. just use a json encoder before using them +in the HTTP request. + +Please note that the data have to be urlencoded at last. (Most libaries will do that automatically) \ No newline at end of file diff --git a/docs/api/datatypes.rst b/docs/api/datatypes.rst new file mode 100644 index 000000000..02e3209cc --- /dev/null +++ b/docs/api/datatypes.rst @@ -0,0 +1,352 @@ +.. _api_datatypes: + +*********************** +API Datatype Definition +*********************** + +Below you find a copy of :file:`module/remote/thriftbackend/pyload.thrift`, which is used to generate the data structs +for various languages. It is also a good overview of avaible methods and return data. + +.. code-block:: c + + .. [[[cog cog.out(open('module/remote/thriftbackend/pyload.thrift', 'rb').read()) ]]] + namespace java org.pyload.thrift + + typedef i32 FileID + typedef i32 PackageID + typedef i32 ResultID + typedef i32 InteractionID + typedef list LinkList + typedef string PluginName + typedef byte Progress + typedef byte Priority + + + enum DownloadStatus { + Finished + Offline, + Online, + Queued, + Skipped, + Waiting, + TempOffline, + Starting, + Failed, + Aborted, + Decrypting, + Custom, + Downloading, + Processing, + Unknown + } + + enum Destination { + Collector, + Queue + } + + // types for user interaction + // some may only be place holder currently not supported + // also all input - output combination are not reasonable, see InteractionManager for further info + enum Input { + NONE, + TEXT, + TEXTBOX, + PASSWORD, + BOOL, // confirm like, yes or no dialog + CLICK, // for positional captchas + CHOICE, // choice from list + MULTIPLE, // multiple choice from list of elements + LIST, // arbitary list of elements + TABLE // table like data structure + } + // more can be implemented by need + + // this describes the type of the outgoing interaction + // ensure they can be logcial or'ed + enum Output { + CAPTCHA = 1, + QUESTION = 2, + NOTIFICATION = 4, + } + + struct DownloadInfo { + 1: FileID fid, + 2: string name, + 3: i64 speed, + 4: i32 eta, + 5: string format_eta, + 6: i64 bleft, + 7: i64 size, + 8: string format_size, + 9: Progress percent, + 10: DownloadStatus status, + 11: string statusmsg, + 12: string format_wait, + 13: i64 wait_until, + 14: PackageID packageID, + 15: string packageName, + 16: PluginName plugin, + } + + struct ServerStatus { + 1: bool pause, + 2: i16 active, + 3: i16 queue, + 4: i16 total, + 5: i64 speed, + 6: bool download, + 7: bool reconnect + } + + struct FileData { + 1: FileID fid, + 2: string url, + 3: string name, + 4: PluginName plugin, + 5: i64 size, + 6: string format_size, + 7: DownloadStatus status, + 8: string statusmsg, + 9: PackageID packageID, + 10: string error, + 11: i16 order + } + + struct PackageData { + 1: PackageID pid, + 2: string name, + 3: string folder, + 4: string site, + 5: string password, + 6: Destination dest, + 7: i16 order, + 8: optional i16 linksdone, + 9: optional i64 sizedone, + 10: optional i64 sizetotal, + 11: optional i16 linkstotal, + 12: optional list links, + 13: optional list fids + } + + struct InteractionTask { + 1: InteractionID iid, + 2: Input input, + 3: list structure, + 4: list preset, + 5: Output output, + 6: list data, + 7: string title, + 8: string description, + 9: string plugin, + } + + struct ConfigItem { + 1: string name, + 2: string long_name, + 3: string description, + 4: string type, + 5: string default_value, + 6: string value, + } + + struct ConfigSection { + 1: string name, + 2: string long_name, + 3: string description, + 4: string long_description, + 5: optional list items, + 6: optional map handler, + } + + struct CaptchaTask { + 1: i16 tid, + 2: binary data, + 3: string type, + 4: string resultType + } + + struct EventInfo { + 1: string eventname, + 2: list event_args, + } + + struct UserData { + 1: string name, + 2: string email, + 3: i32 role, + 4: i32 permission, + 5: string templateName + } + + struct AccountInfo { + 1: PluginName plugin, + 2: string loginname, + 3: bool valid, + 4: i64 validuntil, + 5: i64 trafficleft, + 6: i64 maxtraffic, + 7: bool premium, + 8: bool activated, + 9: map options, + } + + struct ServiceCall { + 1: PluginName plugin, + 2: string func, + 3: string arguments, // empty string or json encoded list + } + + struct OnlineStatus { + 1: string name, + 2: PluginName plugin, + 3: string packagename, + 4: DownloadStatus status, + 5: i64 size, // size <= 0 : unknown + } + + struct OnlineCheck { + 1: ResultID rid, // -1 -> nothing more to get + 2: map data, //url to result + } + + + // exceptions + + exception PackageDoesNotExists{ + 1: PackageID pid + } + + exception FileDoesNotExists{ + 1: FileID fid + } + + exception UserDoesNotExists{ + 1: string user + } + + exception ServiceDoesNotExists{ + 1: string plugin + 2: string func + } + + exception ServiceException{ + 1: string msg + } + + service Pyload { + + //config + string getConfigValue(1: string section, 2: string option), + void setConfigValue(1: string section, 2: string option, 3: string value), + map getConfig(), + map getPluginConfig(), + ConfigSection configureSection(1: string section), + + // server status + void pauseServer(), + void unpauseServer(), + bool togglePause(), + ServerStatus statusServer(), + i64 freeSpace(), + string getServerVersion(), + void kill(), + void restart(), + list getLog(1: i32 offset), + bool isTimeDownload(), + bool isTimeReconnect(), + bool toggleReconnect(), + + // download preparing + + // packagename - urls + map generatePackages(1: LinkList links), + map checkURLs(1: LinkList urls), + map parseURLs(1: string html, 2: string url), + + // parses results and generates packages + OnlineCheck checkOnlineStatus(1: LinkList urls), + OnlineCheck checkOnlineStatusContainer(1: LinkList urls, 2: string filename, 3: binary data) + + // poll results from previosly started online check + OnlineCheck pollResults(1: ResultID rid), + + // downloads - information + list statusDownloads(), + PackageData getPackageData(1: PackageID pid) throws (1: PackageDoesNotExists e), + PackageData getPackageInfo(1: PackageID pid) throws (1: PackageDoesNotExists e), + FileData getFileData(1: FileID fid) throws (1: FileDoesNotExists e), + list getQueue(), + list getCollector(), + list getQueueData(), + list getCollectorData(), + map getPackageOrder(1: Destination destination), + map getFileOrder(1: PackageID pid) + + // downloads - adding/deleting + list generateAndAddPackages(1: LinkList links, 2: Destination dest), + PackageID addPackage(1: string name, 2: LinkList links, 3: Destination dest, 4: string password), + void addFiles(1: PackageID pid, 2: LinkList links), + void uploadContainer(1: string filename, 2: binary data), + void deleteFiles(1: list fids), + void deletePackages(1: list pids), + + // downloads - modifying + void pushToQueue(1: PackageID pid), + void pullFromQueue(1: PackageID pid), + void restartPackage(1: PackageID pid), + void restartFile(1: FileID fid), + void recheckPackage(1: PackageID pid), + void stopAllDownloads(), + void stopDownloads(1: list fids), + void setPackageName(1: PackageID pid, 2: string name), + void movePackage(1: Destination destination, 2: PackageID pid), + void moveFiles(1: list fids, 2: PackageID pid), + void orderPackage(1: PackageID pid, 2: i16 position), + void orderFile(1: FileID fid, 2: i16 position), + void setPackageData(1: PackageID pid, 2: map data) throws (1: PackageDoesNotExists e), + list deleteFinished(), + void restartFailed(), + + //events + list getEvents(1: string uuid) + + //accounts + list getAccounts(1: bool refresh), + list getAccountTypes() + void updateAccount(1: PluginName plugin, 2: string account, 3: string password, 4: map options), + void removeAccount(1: PluginName plugin, 2: string account), + + //auth + bool login(1: string username, 2: string password), + UserData getUserData(1: string username, 2:string password) throws (1: UserDoesNotExists ex), + map getAllUserData(), + + //services + + // servicename : description + map> getServices(), + bool hasService(1: PluginName plugin, 2: string func), + string call(1: ServiceCall info) throws (1: ServiceDoesNotExists ex, 2: ServiceException e), + + + //info + // {plugin: {name: value}} + map> getAllInfo(), + map getInfoByPlugin(1: PluginName plugin), + + //scheduler + + // TODO + + + // User interaction + + //captcha + bool isCaptchaWaiting(), + CaptchaTask getCaptchaTask(1: bool exclusive), + string getCaptchaTaskStatus(1: InteractionID tid), + void setCaptchaResult(1: InteractionID tid, 2: string result), + } + .. [[[end]]] + diff --git a/docs/api/overview.rst b/docs/api/overview.rst new file mode 100644 index 000000000..02cee3e0d --- /dev/null +++ b/docs/api/overview.rst @@ -0,0 +1,35 @@ +.. _overview: + +======================================= +API - Application Programming Interface +======================================= + +From Wikipedia, the free encyclopedia [1]_: + + An application programming interface (API) is a source code based specification intended to be used as an interface + by software components to communicate with each other. An API may include specifications for routines, + data structures, object classes, and variables. + +.. rubric:: Motivation + +The idea of the centralized pyLoad :class:`Api ` is to give uniform access to all integral parts +and plugins in pyLoad, and furthermore to other clients, written in arbitrary programming languages. +Most of the :class:`Api ` functionality is exposed via RPC [2]_ and accessible via thrift [3]_ or +simple JSON objects [4]_. In conclusion the :class:`Api ` is accessible via many programming language, +over network from remote maschines and over browser with javascript. + + +.. rubric:: Contents + +.. toctree:: + + access_api.rst + datatypes.rst + + +.. rubric:: Footnotes + +.. [1] http://en.wikipedia.org/wiki/Application_programming_interface +.. [2] http://en.wikipedia.org/wiki/Remote_procedure_call +.. [3] http://en.wikipedia.org/wiki/Thrift_(protocol) +.. [4] http://en.wikipedia.org/wiki/Json \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 09e4d0c1c..454ed5967 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -65,8 +65,8 @@ v = options.version.split(".") cog.outl("version = '%s'" % ".".join(v[:2])) cog.outl("release = '%s'" % ".".join(v)) ]]]""" -version = '0.4' -release = '0.4.9' +version = '0.5' +release = '0.5.0' # [[[end]]] diff --git a/docs/extend_pyload.rst b/docs/extend_pyload.rst deleted file mode 100755 index 337cb6854..000000000 --- a/docs/extend_pyload.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _extend_pyload: - -******************** -How to extend pyLoad -******************** - -In general there a two different plugin types. These allow everybody to write powerful, modular plugins without knowing -every detail of the pyLoad core. However you should have some basic knowledge of python. - -.. toctree:: - - write_hooks.rst - write_plugins.rst \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 757fd7537..befac0fd2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,27 +1,46 @@ -.. pyLoad documentation master file, created by - sphinx-quickstart on Sat Jun 4 11:54:34 2011. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. pyLoad documentation master file -Welcome to pyLoad's documentation! -================================== +===================== +pyLoad Documentation +===================== -Great that you found your way to the pyLoad documentation! +.. image:: _static/logo.png + :height: 144 + :width: 412 -We have collected some information here to help developer writing plugins and understandig our code. -If you want to help us developing visit us in our IRC channel #pyload on freenode.net or leave a message in our forum. -Contents: +Great that you found your way to the pyLoad [1]_ documentation! + +This is the ultimate document to get started extending or accessing pyLoad in your own way. +We will cover on how to access the API so you can write your own client to pyLoad. The next big part gives you an idea +how to extend pyLoad and write your own powerful plugins, which perfectly integrate into our system. + +The complete pyLoad source and this documentation is available at bitbucket [2]_. If you would like to contribute +come around in our irc channel [3]_ or open a pull request. +In case you still have question, ask them at our forum [4]_ or in our official irc channel at #pyload @ irc.freenode.net + +We wish you the best of luck and happy programming. +-- the pyLoad Team + +Contents +-------- .. toctree:: :maxdepth: 2 - access_api.rst - extend_pyload.rst + api/overview.rst + plugins/overview.rst module_overview.rst .. currentmodule:: module +.. rubric:: Footnotes + +.. [1] http://pyload.org +.. [2] http://pyload.org/irc +.. [3] http://bitbucket.org/spoob/pyload/overview +.. [4] http://forum.pyload.org + ================== * :ref:`genindex` diff --git a/docs/module_overview.rst b/docs/module_overview.rst index 309dccdfd..b8db51538 100644 --- a/docs/module_overview.rst +++ b/docs/module_overview.rst @@ -1,7 +1,8 @@ + Module Overview =============== -You can find an overview of some important classes here: +A little selection of most important modules in pyLoad. .. autosummary:: :toctree: module @@ -10,8 +11,9 @@ You can find an overview of some important classes here: module.plugins.Base.Base module.plugins.Hoster.Hoster module.plugins.Crypter.Crypter - module.plugins.Account.Account module.plugins.Hook.Hook + module.plugins.Account.Account + module.plugins.MultiHoster.MultiHoster module.HookManager.HookManager module.interaction.EventManager.EventManager module.interaction.InteractionManager.InteractionManager diff --git a/docs/plugins/account_plugin.rst b/docs/plugins/account_plugin.rst new file mode 100644 index 000000000..75bf61a75 --- /dev/null +++ b/docs/plugins/account_plugin.rst @@ -0,0 +1,5 @@ +.. _account_plugin: + +Account - Premium Access +======================== + diff --git a/docs/plugins/base_plugin.rst b/docs/plugins/base_plugin.rst new file mode 100644 index 000000000..62ab248ef --- /dev/null +++ b/docs/plugins/base_plugin.rst @@ -0,0 +1,5 @@ +.. _base_plugin: + +Base Plugin - And here it begins... +=================================== + diff --git a/docs/plugins/crypter_plugin.rst b/docs/plugins/crypter_plugin.rst new file mode 100644 index 000000000..d910ec412 --- /dev/null +++ b/docs/plugins/crypter_plugin.rst @@ -0,0 +1,5 @@ +.. _crypter_plugin: + +Crypter - Extract links from pages +================================== + diff --git a/docs/plugins/hook_plugin.rst b/docs/plugins/hook_plugin.rst new file mode 100644 index 000000000..e263ece2e --- /dev/null +++ b/docs/plugins/hook_plugin.rst @@ -0,0 +1,162 @@ +.. _write_hooks: + +Hook - Do everything you want +============================= + +A Hook is a python file which is located at :file:`module/plugins/hooks`. +The :class:`HookManager ` will load it automatically on startup. Only one instance exists +over the complete lifetime of pyload. Your hook can interact on various events called by the :class:`HookManager `, +do something complete autonomic and has full access to the :class:`Api ` and every detail of pyLoad. +The UpdateManager, CaptchaTrader, UnRar and many more are realised as hooks. + +Hook header +----------- + +Your hook needs to subclass :class:`Hook ` and will inherit all of its method, make sure to check its documentation! + +All Hooks should start with something like this: :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + __name__ = "YourHook" + __version__ = "0.1" + __description__ = "Does really cool stuff" + __config__ = [ ("activated" , "bool" , "Activated" , "True" ) ] + __threaded__ = ["downloadFinished"] + __author_name__ = ("Me") + __author_mail__ = ("me@has-no-mail.com") + +All meta-data is defined in the header, you need at least one option at ``__config__`` so the user can toggle your +hook on and off. Dont't overwrite the ``init`` method if not neccesary, use ``setup`` instead. + +Using the Config +---------------- + +We are taking a closer look at the ``__config__`` parameter. +You can add more config values as desired by adding tuples of the following format to the config list: ``("name", "type", "description", "default value")``. +When everything went right you can access the config values with ``self.getConfig(name)`` and ``self.setConfig(name,value``. + + +Interacting on Events +--------------------- + +The next step is to think about where your Hook action takes places. + +The easiest way is to overwrite specific methods defined by the :class:`Hook ` base class. +The name is indicating when the function gets called. +See :class:`Hook ` page for a complete listing. + +You should be aware of the arguments the Hooks are called with, whether its a :class:`PyFile ` +or :class:`PyPackage ` you should read its related documentation to know how to access her great power and manipulate them. + +A basic excerpt would look like: :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + + def coreReady(self): + print "Yay, the core is ready let's do some work." + + def downloadFinished(self, pyfile): + print "A Download just finished." + +Another important feature to mention can be seen at the ``__threaded__`` parameter. Function names listed will be executed +in a thread, in order to not block the main thread. This should be used for all kind of longer processing tasks. + +Another and more flexible and powerful way is to use event listener. +All hook methods exists as event and very useful additional events are dispatched by the core. For a little overview look +at :class:`HookManager `. Keep in mind that you can define own events and other people may listen on them. + +For your convenience it's possible to register listeners automatical via the ``event_map`` attribute. +It requires a `dict` that maps event names to function names or a `list` of function names. It's important that all names are strings :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + event_map = {"downloadFinished" : "doSomeWork", + "allDownloadsFnished": "someMethod", + "coreReady": "initialize"} + + def initialize(self): + print "Initialized." + + def doSomeWork(self, pyfile): + print "This is equivalent to the above example." + + def someMethod(self): + print "The underlying event (allDownloadsFinished) for this method is not available through the base class" + +An advantage of the event listener is that you are able to register and remove the listeners at runtime. +Use `self.manager.addEvent("name", function)`, `self.manager.removeEvent("name", function)` and see doc for +:class:`HookManager `. Contrary to ``event_map``, ``function`` has to be a reference +and **not** a `string`. + +We introduced events because it scales better if there a a huge amount of events and hooks. So all future interaction will be exclusive +available as event and not accessible through overwriting hook methods. However you can safely do this, it will not be removed and is easier to implement. + + +Providing RPC services +---------------------- + +You may noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. +So probably clients want to be able to interact with your hook to request it's state or invoke some action. + +Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: + + from module.plugins.Hook import Hook, Expose + + class YourHook(Hook): + """ + Your Hook code here. + """ + + @Expose + def invoke(self, arg): + print "Invoked with", arg + +Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. +Here is a basic example: :: + + #Assuming client is a ThriftClient or Api object + + print client.getServices() + print client.call(ServiceCall("YourHook", "invoke", "an argument")) + +Providing status information +---------------------------- +Your hook can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. + +Just store everything in ``self.info``. :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + + def setup(self): + self.info = {"running": False} + + def coreReady(self): + self.info["running"] = True + +Usable with: :: + + #Assuming client is a ThriftClient or Api object + + print client.getAllInfo() + +Example +------- + Sorry but you won't find an example here ;-) + + Look at :file:`module/plugins/hooks` and you will find plenty examples there. diff --git a/docs/plugins/hoster_plugin.rst b/docs/plugins/hoster_plugin.rst new file mode 100644 index 000000000..59f35f5cb --- /dev/null +++ b/docs/plugins/hoster_plugin.rst @@ -0,0 +1,102 @@ +.. _hoster_plugin: + +Hoster - Load files to disk +=========================== + +A Plugin is a python file located at one of the subfolders in :file:`module/plugins/`. Either :file:`hoster`, :file:`crypter` +or :file:`container`, depending of it's type. + +There are three kinds of different plugins: **Hoster**, **Crypter**, **Container**. +All kind of plugins inherit from the base :class:`Plugin `. You should know its +convenient methods, they make your work easier ;-) + +Every plugin defines a ``__pattern__`` and when the user adds urls, every url is matched against the pattern defined in +the plugin. In case the ``__pattern__`` matched on the url the plugin will be assigned to handle it and instanciated when +pyLoad begins to download/decrypt the url. + +Plugin header +------------- + +How basic hoster plugin header could look like: :: + + from module.plugin.Hoster import Hoster + + class MyFileHoster(Hoster): + __version__ = "0.1" + __pattern__ = r"http://myfilehoster.example.com/file_id/[0-9]+" + __config__ = [] + +You have to define these meta-data, ``__pattern__`` has to be a regexp that sucessfully compiles with +``re.compile(__pattern__)``. + +Just like :ref:`write_hooks` you can add and use config values exatly the same way. +If you want a Crypter or Container plugin, just replace the word Hoster with your desired plugin type. + + +Hoster plugins +-------------- + +We head to the next important section, the ``process`` method of your plugin. +In fact the ``process`` method is the only functionality your plugin has to provide, but its always a good idea to split up tasks to not produce spaghetti code. +An example ``process`` function could look like this :: + + from module.plugin.Hoster import Hoster + + class MyFileHoster(Hoster): + """ + plugin code + """ + + def process(self, pyfile): + html = self.load(pyfile.url) # load the content of the orginal pyfile.url to html + + # parse the name from the site and set attribute in pyfile + pyfile.name = self.myFunctionToParseTheName(html) + parsed_url = self.myFunctionToParseUrl(html) + + # download the file, destination is determined by pyLoad + self.download(parsed_url) + +You need to know about the :class:`PyFile ` class, since an instance of it is given as parameter to every pyfile. +Some tasks your plugin should handle: proof if file is online, get filename, wait if needed, download the file, etc.. + +Wait times +__________ + +Some hoster require you to wait a specific time. Just set the time with ``self.setWait(seconds)`` or +``self.setWait(seconds, True)`` if you want pyLoad to perform a reconnect if needed. + +Captcha decrypting +__________________ + +To handle captcha input just use ``self.decryptCaptcha(url, ...)``, it will be send to clients +or handled by :class:`Hook ` plugins + +Crypter +------- + +What about Decrypter and Container plugins? +Well, they work nearly the same, only that the function they have to provide is named ``decrypt`` + +Example: :: + + from module.plugin.Crypter import Crypter + + class MyFileCrypter(Crypter): + """ + plugin code + """ + def decrypt(self, pyfile): + + urls = ["http://get.pyload.org/src", "http://get.pyload.org/debian", "http://get.pyload.org/win"] + + self.packages.append(("pyLoad packages", urls, "pyLoad packages")) # urls list of urls + +They can access all the methods from :class:`Plugin `, but the important thing is they +have to append all packages they parsed to the `self.packages` list. Simply append tuples with `(name, urls, folder)`, +where urls is the list of urls contained in the packages. Thats all of your work, pyLoad will know what to do with them. + +Examples +-------- + +Best examples are already existing plugins in :file:`module/plugins/`. \ No newline at end of file diff --git a/docs/plugins/overview.rst b/docs/plugins/overview.rst new file mode 100755 index 000000000..23913b787 --- /dev/null +++ b/docs/plugins/overview.rst @@ -0,0 +1,31 @@ +.. _overview: + +================ +Extending pyLoad +================ + +.. pull-quote:: + Any sufficiently advanced technology is indistinguishable from magic. + + -- Arthur C. Clarke + + +.. rubric:: Motivation + +pyLoad offers an comfortable and powerful plugin system to make extending possible. With it you only need to have some +python knowledge and can just start right away writing your own plugins. This document gives you an overwiew about the +conceptual part. You should not left out the `Base` part, since it contains basic functionality for all plugins types. + +.. rubric:: Contents + +.. toctree:: + + base_plugin.rst + crypter_plugin.rst + hoster_plugin.rst + account_plugin.rst + hook_plugin.rst + + + +.. rubric:: Footnotes \ No newline at end of file diff --git a/docs/write_hooks.rst b/docs/write_hooks.rst deleted file mode 100644 index dd60367b7..000000000 --- a/docs/write_hooks.rst +++ /dev/null @@ -1,162 +0,0 @@ -.. _write_hooks: - -Hooks -===== - -A Hook is a python file which is located at :file:`module/plugins/hooks`. -The :class:`HookManager ` will load it automatically on startup. Only one instance exists -over the complete lifetime of pyload. Your hook can interact on various events called by the :class:`HookManager `, -do something complete autonomic and has full access to the :class:`Api ` and every detail of pyLoad. -The UpdateManager, CaptchaTrader, UnRar and many more are realised as hooks. - -Hook header ------------ - -Your hook needs to subclass :class:`Hook ` and will inherit all of its method, make sure to check its documentation! - -All Hooks should start with something like this: :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - __name__ = "YourHook" - __version__ = "0.1" - __description__ = "Does really cool stuff" - __config__ = [ ("activated" , "bool" , "Activated" , "True" ) ] - __threaded__ = ["downloadFinished"] - __author_name__ = ("Me") - __author_mail__ = ("me@has-no-mail.com") - -All meta-data is defined in the header, you need at least one option at ``__config__`` so the user can toggle your -hook on and off. Dont't overwrite the ``init`` method if not neccesary, use ``setup`` instead. - -Using the Config ----------------- - -We are taking a closer look at the ``__config__`` parameter. -You can add more config values as desired by adding tuples of the following format to the config list: ``("name", "type", "description", "default value")``. -When everything went right you can access the config values with ``self.getConfig(name)`` and ``self.setConfig(name,value``. - - -Interacting on Events ---------------------- - -The next step is to think about where your Hook action takes places. - -The easiest way is to overwrite specific methods defined by the :class:`Hook ` base class. -The name is indicating when the function gets called. -See :class:`Hook ` page for a complete listing. - -You should be aware of the arguments the Hooks are called with, whether its a :class:`PyFile ` -or :class:`PyPackage ` you should read its related documentation to know how to access her great power and manipulate them. - -A basic excerpt would look like: :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - """ - Your Hook code here. - """ - - def coreReady(self): - print "Yay, the core is ready let's do some work." - - def downloadFinished(self, pyfile): - print "A Download just finished." - -Another important feature to mention can be seen at the ``__threaded__`` parameter. Function names listed will be executed -in a thread, in order to not block the main thread. This should be used for all kind of longer processing tasks. - -Another and more flexible and powerful way is to use event listener. -All hook methods exists as event and very useful additional events are dispatched by the core. For a little overview look -at :class:`HookManager `. Keep in mind that you can define own events and other people may listen on them. - -For your convenience it's possible to register listeners automatical via the ``event_map`` attribute. -It requires a `dict` that maps event names to function names or a `list` of function names. It's important that all names are strings :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - """ - Your Hook code here. - """ - event_map = {"downloadFinished" : "doSomeWork", - "allDownloadsFnished": "someMethod", - "coreReady": "initialize"} - - def initialize(self): - print "Initialized." - - def doSomeWork(self, pyfile): - print "This is equivalent to the above example." - - def someMethod(self): - print "The underlying event (allDownloadsFinished) for this method is not available through the base class" - -An advantage of the event listener is that you are able to register and remove the listeners at runtime. -Use `self.manager.addEvent("name", function)`, `self.manager.removeEvent("name", function)` and see doc for -:class:`HookManager `. Contrary to ``event_map``, ``function`` has to be a reference -and **not** a `string`. - -We introduced events because it scales better if there a a huge amount of events and hooks. So all future interaction will be exclusive -available as event and not accessible through overwriting hook methods. However you can safely do this, it will not be removed and is easier to implement. - - -Providing RPC services ----------------------- - -You may noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. -So probably clients want to be able to interact with your hook to request it's state or invoke some action. - -Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: - - from module.plugins.Hook import Hook, Expose - - class YourHook(Hook): - """ - Your Hook code here. - """ - - @Expose - def invoke(self, arg): - print "Invoked with", arg - -Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. -Here is a basic example: :: - - #Assuming client is a ThriftClient or Api object - - print client.getServices() - print client.call(ServiceCall("YourHook", "invoke", "an argument")) - -Providing status information ----------------------------- -Your hook can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. - -Just store everything in ``self.info``. :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - """ - Your Hook code here. - """ - - def setup(self): - self.info = {"running": False} - - def coreReady(self): - self.info["running"] = True - -Usable with: :: - - #Assuming client is a ThriftClient or Api object - - print client.getAllInfo() - -Example -------- - Sorry but you won't find an example here ;-) - - Look at :file:`module/plugins/hooks` and you will find plenty examples there. diff --git a/docs/write_plugins.rst b/docs/write_plugins.rst deleted file mode 100644 index b513a5978..000000000 --- a/docs/write_plugins.rst +++ /dev/null @@ -1,103 +0,0 @@ -.. _write_plugins: - -Plugins -======= - -A Plugin is a python file located at one of the subfolders in :file:`module/plugins/`. Either :file:`hoster`, :file:`crypter` -or :file:`container`, depending of it's type. - -There are three kinds of different plugins: **Hoster**, **Crypter**, **Container**. -All kind of plugins inherit from the base :class:`Plugin `. You should know its -convenient methods, they make your work easier ;-) - -Every plugin defines a ``__pattern__`` and when the user adds urls, every url is matched against the pattern defined in -the plugin. In case the ``__pattern__`` matched on the url the plugin will be assigned to handle it and instanciated when -pyLoad begins to download/decrypt the url. - -Plugin header -------------- - -How basic hoster plugin header could look like: :: - - from module.plugin.Hoster import Hoster - - class MyFileHoster(Hoster): - __name__ = "MyFileHoster" - __version__ = "0.1" - __pattern__ = r"http://myfilehoster.example.com/file_id/[0-9]+" - __config__ = [] - -You have to define these meta-data, ``__pattern__`` has to be a regexp that sucessfully compiles with -``re.compile(__pattern__)``. - -Just like :ref:`write_hooks` you can add and use config values exatly the same way. -If you want a Crypter or Container plugin, just replace the word Hoster with your desired plugin type. - - -Hoster plugins --------------- - -We head to the next important section, the ``process`` method of your plugin. -In fact the ``process`` method is the only functionality your plugin has to provide, but its always a good idea to split up tasks to not produce spaghetti code. -An example ``process`` function could look like this :: - - from module.plugin.Hoster import Hoster - - class MyFileHoster(Hoster): - """ - plugin code - """ - - def process(self, pyfile): - html = self.load(pyfile.url) # load the content of the orginal pyfile.url to html - - # parse the name from the site and set attribute in pyfile - pyfile.name = self.myFunctionToParseTheName(html) - parsed_url = self.myFunctionToParseUrl(html) - - # download the file, destination is determined by pyLoad - self.download(parsed_url) - -You need to know about the :class:`PyFile ` class, since an instance of it is given as parameter to every pyfile. -Some tasks your plugin should handle: proof if file is online, get filename, wait if needed, download the file, etc.. - -Wait times -__________ - -Some hoster require you to wait a specific time. Just set the time with ``self.setWait(seconds)`` or -``self.setWait(seconds, True)`` if you want pyLoad to perform a reconnect if needed. - -Captcha decrypting -__________________ - -To handle captcha input just use ``self.decryptCaptcha(url, ...)``, it will be send to clients -or handled by :class:`Hook ` plugins - -Crypter -------- - -What about Decrypter and Container plugins? -Well, they work nearly the same, only that the function they have to provide is named ``decrypt`` - -Example: :: - - from module.plugin.Crypter import Crypter - - class MyFileCrypter(Crypter): - """ - plugin code - """ - def decrypt(self, pyfile): - - urls = ["http://get.pyload.org/src", "http://get.pyload.org/debian", "http://get.pyload.org/win"] - - self.packages.append(("pyLoad packages", urls, "pyLoad packages")) # urls list of urls - -They can access all the methods from :class:`Plugin `, but the important thing is they -have to append all packages they parsed to the `self.packages` list. Simply append tuples with `(name, urls, folder)`, -where urls is the list of urls contained in the packages. Thats all of your work, pyLoad will know what to do with them. - -Examples --------- - -Best examples are already existing plugins in :file:`module/plugins/`. \ No newline at end of file diff --git a/module/HookManager.py b/module/HookManager.py index d0ceb89b2..51bc706ca 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -63,7 +63,7 @@ class HookManager: func = getattr(hook, f) return func(*args) except Exception, e: - plugin.logError(_("Error executing %s" % event), e) + hook.logError(_("Error when executing %s" % f), e) if self.core.debug: print_exc() diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index fe8075539..0d5fc59c9 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -17,9 +17,9 @@ @author: RaNaN """ -from os import remove, fsync +from os import remove from os.path import dirname -from time import sleep, time +from time import time from shutil import move from logging import getLogger @@ -28,7 +28,7 @@ import pycurl from HTTPChunk import ChunkInfo, HTTPChunk from HTTPRequest import BadHeader -from module.plugins.Plugin import Abort +from module.plugins.Hoster import Abort from module.utils import save_join, fs_encode class HTTPDownload(): diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index d4c33bbff..cd13dd01f 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -25,7 +25,7 @@ from httplib import responses from logging import getLogger from cStringIO import StringIO -from module.plugins.Plugin import Abort +from module.plugins.Hoster import Abort def myquote(url): return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") diff --git a/module/plugins/Account.py b/module/plugins/Account.py index 59ce87ed2..e5b90d95e 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -4,12 +4,13 @@ from time import time from traceback import print_exc from threading import RLock -from Plugin import Base from module.utils import compare_time, parseFileSize, lock from module.config.converter import from_string from module.Api import AccountInfo from module.network.CookieJar import CookieJar +from Base import Base + class WrongPassword(Exception): pass diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index 5d164da64..3e423881e 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -23,7 +23,10 @@ class Package: return self.name == other.name and self.urls == other.urls def __repr__(self): - return " LinkList @@ -334,6 +333,6 @@ service Pyload { //captcha bool isCaptchaWaiting(), CaptchaTask getCaptchaTask(1: bool exclusive), - string getCaptchaTaskStatus(1: TaskID tid), - void setCaptchaResult(1: TaskID tid, 2: string result), + string getCaptchaTaskStatus(1: InteractionID tid), + void setCaptchaResult(1: InteractionID tid, 2: string result), } diff --git a/module/threads/DecrypterThread.py b/module/threads/DecrypterThread.py index 8edb97c34..ce3c8cd83 100644 --- a/module/threads/DecrypterThread.py +++ b/module/threads/DecrypterThread.py @@ -72,7 +72,7 @@ class DecrypterThread(BaseThread): self.log.info(_("Decrypted %(count)d links into package %(name)s") % {"count": len(urls), "name": pack.name}) self.m.core.api.addFiles(self.pid, urls) - for p in pack_names: + for p in pack_names.itervalues(): self.m.core.api.addPackage(p.name, p.urls, p.dest, pack.password) if not result: diff --git a/pavement.py b/pavement.py index 8ebd5bfc5..f3e8651c5 100644 --- a/pavement.py +++ b/pavement.py @@ -5,6 +5,21 @@ from paver.easy import * from paver.setuputils import setup from paver.doctools import cog +import fnmatch + +# patch to let it support list of patterns +def new_fnmatch(self, pattern): + if type(pattern) == list: + for p in pattern: + if fnmatch.fnmatch(self.name, p): + return True + return False + else: + return fnmatch.fnmatch(self.name, pattern) + +path.fnmatch = new_fnmatch + + import sys import re from urllib import urlretrieve @@ -86,7 +101,7 @@ options( virtual="virtualenv2", ), cog=Bunch( - pattern="*.py", + pattern=["*.py", "*.rst"], ) ) -- cgit v1.2.3