From 35742c2cb023ac49ab3056752d2040cdb030cc2b Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 1 Jan 2012 13:36:59 +0100 Subject: Happy new Year ! --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'docs') diff --git a/docs/conf.py b/docs/conf.py index 9d2cf98f9..09e4d0c1c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,7 +52,7 @@ master_doc = 'index' # General information about the project. project = u'pyLoad' -copyright = u'2011, pyLoad Team' +copyright = u'2012, pyLoad Team' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the -- cgit v1.2.3 From 4a3a81b63cd85cc3dcd9669868a2079da65838a2 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 3 Jan 2012 20:41:23 +0100 Subject: fixes for old style decrypter --- docs/module_overview.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'docs') diff --git a/docs/module_overview.rst b/docs/module_overview.rst index d51202c88..309dccdfd 100644 --- a/docs/module_overview.rst +++ b/docs/module_overview.rst @@ -7,11 +7,14 @@ You can find an overview of some important classes here: :toctree: module module.Api.Api - module.plugins.Plugin.Base - module.plugins.Plugin.Plugin + module.plugins.Base.Base + module.plugins.Hoster.Hoster module.plugins.Crypter.Crypter module.plugins.Account.Account module.plugins.Hook.Hook module.HookManager.HookManager + module.interaction.EventManager.EventManager + module.interaction.InteractionManager.InteractionManager + module.interaction.InteractionTask.InteractionTask module.PyFile.PyFile module.PyPackage.PyPackage -- cgit v1.2.3 From 1bb6ebf544b43cacf7c0755c5a8608b79b95e2d6 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 7 Jan 2012 20:11:16 +0100 Subject: MultiHoster plugin type, some fixes, new documentation structure --- docs/access_api.rst | 121 -------------- docs/api/access_api.rst | 122 ++++++++++++++ docs/api/datatypes.rst | 352 ++++++++++++++++++++++++++++++++++++++++ docs/api/overview.rst | 35 ++++ docs/conf.py | 4 +- docs/extend_pyload.rst | 13 -- docs/index.rst | 43 +++-- docs/module_overview.rst | 6 +- docs/plugins/account_plugin.rst | 5 + docs/plugins/base_plugin.rst | 5 + docs/plugins/crypter_plugin.rst | 5 + docs/plugins/hook_plugin.rst | 162 ++++++++++++++++++ docs/plugins/hoster_plugin.rst | 102 ++++++++++++ docs/plugins/overview.rst | 31 ++++ docs/write_hooks.rst | 162 ------------------ docs/write_plugins.rst | 103 ------------ 16 files changed, 856 insertions(+), 415 deletions(-) delete mode 100644 docs/access_api.rst create mode 100644 docs/api/access_api.rst create mode 100644 docs/api/datatypes.rst create mode 100644 docs/api/overview.rst delete mode 100755 docs/extend_pyload.rst create mode 100644 docs/plugins/account_plugin.rst create mode 100644 docs/plugins/base_plugin.rst create mode 100644 docs/plugins/crypter_plugin.rst create mode 100644 docs/plugins/hook_plugin.rst create mode 100644 docs/plugins/hoster_plugin.rst create mode 100755 docs/plugins/overview.rst delete mode 100644 docs/write_hooks.rst delete mode 100644 docs/write_plugins.rst (limited to 'docs') diff --git a/docs/access_api.rst b/docs/access_api.rst deleted file mode 100644 index df69da8b2..000000000 --- a/docs/access_api.rst +++ /dev/null @@ -1,121 +0,0 @@ -.. _access_api: - -********************* -How to access the API -********************* - -pyLoad has a very powerfull API with can be accessed in several ways. - -Overview --------- - -First of all, you need to know what you can do with our API. It lets you do all common task like -retrieving download status, manage queue, manage accounts, modify config and so on. - -This document is not intended to explain every function in detail, for a complete listing -see :class:`Api `. - -Of course its possible to access the ``core.api`` attribute in plugins and hooks, but much more -interesting is the possibillity to call function from different programs written in many different languages. - -pyLoad uses thrift as backend and provides its :class:`Api ` as service. -More information about thrift can be found here http://wiki.apache.org/thrift/. - - -Using Thrift ------------- - -Every thrift service has to define all data structures and declare every method which should be usable via rpc. -This file is located :file:`module/remote/thriftbackend/pyload.thrift`, its very helpful to inform about -arguments and detailed structure of return types. However it does not contain any information about what the functions does. - -Assuming you want to use the API in any other language than python than check if it is -supported here http://wiki.apache.org/thrift/LibraryFeatures?action=show&redirect=LanguageSupport. - -Now install thrift, for instructions see http://wiki.apache.org/thrift/ThriftInstallation. -If every thing went fine you are ready to generate the method stubs, the command basically looks like this. :: - - $ thrift --gen (language) pyload.thrift - -You find now a directory named :file:`gen-(language)`. For instruction how to use the generated files consider the docs -at the thrift wiki and the examples here http://wiki.apache.org/thrift/ThriftUsage. - - -======= -Example -======= -In case you want to use python, pyload has already all files included to access the api over rpc. - -A basic script that prints out some information: :: - - from module.remote.thriftbackend.ThriftClient import ThriftClient, WrongLogin - - try: - client = ThriftClient(host="127.0.0.1", port=7227, user="User", password="yourpw") - except: - print "Login was wrong" - exit() - - print "Server version:", client.getServerVersion() - print client.statusDownloads() - q = client.getQueue() - for p in q: - data = client.getPackageData(p.pid) - print "Package Name: ", data.name - -That's all for now, pretty easy isn't it? -If you still have open questions come around in irc or post them at our pyload forum. - - -Using HTTP/JSON ---------------- - -Another maybe easier way, which does not require much setup is to access the JSON Api via HTTP. -For this reason the webinterface must be enabled. - -===== -Login -===== - -First you need to authenticate, if you using this within the webinterface and the user is logged the API is also accessible, -since they share the same cookie/session. - -However, if you are building a external client and want to authenticate manually -you have to send your credentials ``username`` and ``password`` as -POST parameter to ``http://pyload-core/api/login``. - -The result will be your session id. If you are using cookies, it will be set and you can use the API now. -In case you dont't have cookies enabled you can pass the session id as ``session`` POST parameter -so pyLoad can authenticate you. - -=============== -Calling Methods -=============== - -In general you can use any method listed at the :class:`Api ` documentation, which is also available to -the thriftbackend. - -Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address -or hostname including the webinterface port. By default on local access this would be `localhost:8000`. - -The return value will be formatted in JSON, complex data types as dictionaries. -As mentionted above for a documentation about the return types look at the thrift specification file :file:`module/remote/thriftbackend/pyload.thrift`. - -================== -Passing parameters -================== - -To pass arguments you have two choices. -Either use positional arguments, eg ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword arguments -supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names in the :class:`Api ` -documentation. - -It is important that *all* arguments are in JSON format. So ``http://pyload-core/api/getFileData/1`` is valid because -1 represents an integer in json format. On the other hand if the method is expecting strings, this would be correct: -``http://pyload-core/api/getUserData/"username"/"password"``. - -Strings are wrapped in double qoutes, because `"username"` represents a string in json format. It's not limited to strings and intergers, -every container type like lists and dicts are possible. You usually don't have to convert them. just use a json encoder before using them -in the HTTP request. - -Please note that the data have to be urlencoded at last. (Most libaries will do that automatically) \ No newline at end of file diff --git a/docs/api/access_api.rst b/docs/api/access_api.rst new file mode 100644 index 000000000..efa1ae3fc --- /dev/null +++ b/docs/api/access_api.rst @@ -0,0 +1,122 @@ +.. _access_api: + +********************* +How to access the API +********************* + +pyLoad has a very powerfull API with can be accessed in several ways. + +Overview +-------- + +First of all, you need to know what you can do with our API. It lets you do all common task like +retrieving download status, manage queue, manage accounts, modify config and so on. + +This document is not intended to explain every function in detail, for a complete listing +see :class:`Api `. + +Of course its possible to access the ``core.api`` attribute in plugins and hooks, but much more +interesting is the possibillity to call function from different programs written in many different languages. + +pyLoad uses thrift as backend and provides its :class:`Api ` as service. +More information about thrift can be found here http://wiki.apache.org/thrift/. + + +Using Thrift +------------ + +Every thrift service has to define all data structures and declare every method which should be usable via rpc. +This file is located at :file:`module/remote/thriftbackend/pyload.thrift`, its very helpful to inform about +arguments and detailed structure of return types. However it does not contain any information about what the functions does. +You can also look at it :doc:`here ` + +Assuming you want to use the API in any other language than python than check if it is +supported here http://wiki.apache.org/thrift/LibraryFeatures?action=show&redirect=LanguageSupport. + +Now install thrift, for instructions see http://wiki.apache.org/thrift/ThriftInstallation. +If every thing went fine you are ready to generate the method stubs, the command basically looks like this. :: + + $ thrift --gen (language) pyload.thrift + +You find now a directory named :file:`gen-(language)`. For instruction how to use the generated files consider the docs +at the thrift wiki and the examples here http://wiki.apache.org/thrift/ThriftUsage. + + +======= +Example +======= +In case you want to use python, pyload has already all files included to access the api over rpc. + +A basic script that prints out some information: :: + + from module.remote.thriftbackend.ThriftClient import ThriftClient, WrongLogin + + try: + client = ThriftClient(host="127.0.0.1", port=7227, user="User", password="yourpw") + except: + print "Login was wrong" + exit() + + print "Server version:", client.getServerVersion() + print client.statusDownloads() + q = client.getQueue() + for p in q: + data = client.getPackageData(p.pid) + print "Package Name: ", data.name + +That's all for now, pretty easy isn't it? +If you still have open questions come around in irc or post them at our pyload forum. + + +Using HTTP/JSON +--------------- + +Another maybe easier way, which does not require much setup is to access the JSON Api via HTTP. +For this reason the webinterface must be enabled. + +===== +Login +===== + +First you need to authenticate, if you using this within the webinterface and the user is logged the API is also accessible, +since they share the same cookie/session. + +However, if you are building a external client and want to authenticate manually +you have to send your credentials ``username`` and ``password`` as +POST parameter to ``http://pyload-core/api/login``. + +The result will be your session id. If you are using cookies, it will be set and you can use the API now. +In case you dont't have cookies enabled you can pass the session id as ``session`` POST parameter +so pyLoad can authenticate you. + +=============== +Calling Methods +=============== + +In general you can use any method listed at the :class:`Api ` documentation, which is also available to +the thriftbackend. + +Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address +or hostname including the webinterface port. By default on local access this would be `localhost:8000`. + +The return value will be formatted in JSON, complex data types as dictionaries. +As mentionted above for a documentation about the return types look at the thrift specification file :file:`module/remote/thriftbackend/pyload.thrift`. + +================== +Passing parameters +================== + +To pass arguments you have two choices. +Either use positional arguments, eg ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword arguments +supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names in the :class:`Api ` +documentation. + +It is important that *all* arguments are in JSON format. So ``http://pyload-core/api/getFileData/1`` is valid because +1 represents an integer in json format. On the other hand if the method is expecting strings, this would be correct: +``http://pyload-core/api/getUserData/"username"/"password"``. + +Strings are wrapped in double qoutes, because `"username"` represents a string in json format. It's not limited to strings and intergers, +every container type like lists and dicts are possible. You usually don't have to convert them. just use a json encoder before using them +in the HTTP request. + +Please note that the data have to be urlencoded at last. (Most libaries will do that automatically) \ No newline at end of file diff --git a/docs/api/datatypes.rst b/docs/api/datatypes.rst new file mode 100644 index 000000000..02e3209cc --- /dev/null +++ b/docs/api/datatypes.rst @@ -0,0 +1,352 @@ +.. _api_datatypes: + +*********************** +API Datatype Definition +*********************** + +Below you find a copy of :file:`module/remote/thriftbackend/pyload.thrift`, which is used to generate the data structs +for various languages. It is also a good overview of avaible methods and return data. + +.. code-block:: c + + .. [[[cog cog.out(open('module/remote/thriftbackend/pyload.thrift', 'rb').read()) ]]] + namespace java org.pyload.thrift + + typedef i32 FileID + typedef i32 PackageID + typedef i32 ResultID + typedef i32 InteractionID + typedef list LinkList + typedef string PluginName + typedef byte Progress + typedef byte Priority + + + enum DownloadStatus { + Finished + Offline, + Online, + Queued, + Skipped, + Waiting, + TempOffline, + Starting, + Failed, + Aborted, + Decrypting, + Custom, + Downloading, + Processing, + Unknown + } + + enum Destination { + Collector, + Queue + } + + // types for user interaction + // some may only be place holder currently not supported + // also all input - output combination are not reasonable, see InteractionManager for further info + enum Input { + NONE, + TEXT, + TEXTBOX, + PASSWORD, + BOOL, // confirm like, yes or no dialog + CLICK, // for positional captchas + CHOICE, // choice from list + MULTIPLE, // multiple choice from list of elements + LIST, // arbitary list of elements + TABLE // table like data structure + } + // more can be implemented by need + + // this describes the type of the outgoing interaction + // ensure they can be logcial or'ed + enum Output { + CAPTCHA = 1, + QUESTION = 2, + NOTIFICATION = 4, + } + + struct DownloadInfo { + 1: FileID fid, + 2: string name, + 3: i64 speed, + 4: i32 eta, + 5: string format_eta, + 6: i64 bleft, + 7: i64 size, + 8: string format_size, + 9: Progress percent, + 10: DownloadStatus status, + 11: string statusmsg, + 12: string format_wait, + 13: i64 wait_until, + 14: PackageID packageID, + 15: string packageName, + 16: PluginName plugin, + } + + struct ServerStatus { + 1: bool pause, + 2: i16 active, + 3: i16 queue, + 4: i16 total, + 5: i64 speed, + 6: bool download, + 7: bool reconnect + } + + struct FileData { + 1: FileID fid, + 2: string url, + 3: string name, + 4: PluginName plugin, + 5: i64 size, + 6: string format_size, + 7: DownloadStatus status, + 8: string statusmsg, + 9: PackageID packageID, + 10: string error, + 11: i16 order + } + + struct PackageData { + 1: PackageID pid, + 2: string name, + 3: string folder, + 4: string site, + 5: string password, + 6: Destination dest, + 7: i16 order, + 8: optional i16 linksdone, + 9: optional i64 sizedone, + 10: optional i64 sizetotal, + 11: optional i16 linkstotal, + 12: optional list links, + 13: optional list fids + } + + struct InteractionTask { + 1: InteractionID iid, + 2: Input input, + 3: list structure, + 4: list preset, + 5: Output output, + 6: list data, + 7: string title, + 8: string description, + 9: string plugin, + } + + struct ConfigItem { + 1: string name, + 2: string long_name, + 3: string description, + 4: string type, + 5: string default_value, + 6: string value, + } + + struct ConfigSection { + 1: string name, + 2: string long_name, + 3: string description, + 4: string long_description, + 5: optional list items, + 6: optional map handler, + } + + struct CaptchaTask { + 1: i16 tid, + 2: binary data, + 3: string type, + 4: string resultType + } + + struct EventInfo { + 1: string eventname, + 2: list event_args, + } + + struct UserData { + 1: string name, + 2: string email, + 3: i32 role, + 4: i32 permission, + 5: string templateName + } + + struct AccountInfo { + 1: PluginName plugin, + 2: string loginname, + 3: bool valid, + 4: i64 validuntil, + 5: i64 trafficleft, + 6: i64 maxtraffic, + 7: bool premium, + 8: bool activated, + 9: map options, + } + + struct ServiceCall { + 1: PluginName plugin, + 2: string func, + 3: string arguments, // empty string or json encoded list + } + + struct OnlineStatus { + 1: string name, + 2: PluginName plugin, + 3: string packagename, + 4: DownloadStatus status, + 5: i64 size, // size <= 0 : unknown + } + + struct OnlineCheck { + 1: ResultID rid, // -1 -> nothing more to get + 2: map data, //url to result + } + + + // exceptions + + exception PackageDoesNotExists{ + 1: PackageID pid + } + + exception FileDoesNotExists{ + 1: FileID fid + } + + exception UserDoesNotExists{ + 1: string user + } + + exception ServiceDoesNotExists{ + 1: string plugin + 2: string func + } + + exception ServiceException{ + 1: string msg + } + + service Pyload { + + //config + string getConfigValue(1: string section, 2: string option), + void setConfigValue(1: string section, 2: string option, 3: string value), + map getConfig(), + map getPluginConfig(), + ConfigSection configureSection(1: string section), + + // server status + void pauseServer(), + void unpauseServer(), + bool togglePause(), + ServerStatus statusServer(), + i64 freeSpace(), + string getServerVersion(), + void kill(), + void restart(), + list getLog(1: i32 offset), + bool isTimeDownload(), + bool isTimeReconnect(), + bool toggleReconnect(), + + // download preparing + + // packagename - urls + map generatePackages(1: LinkList links), + map checkURLs(1: LinkList urls), + map parseURLs(1: string html, 2: string url), + + // parses results and generates packages + OnlineCheck checkOnlineStatus(1: LinkList urls), + OnlineCheck checkOnlineStatusContainer(1: LinkList urls, 2: string filename, 3: binary data) + + // poll results from previosly started online check + OnlineCheck pollResults(1: ResultID rid), + + // downloads - information + list statusDownloads(), + PackageData getPackageData(1: PackageID pid) throws (1: PackageDoesNotExists e), + PackageData getPackageInfo(1: PackageID pid) throws (1: PackageDoesNotExists e), + FileData getFileData(1: FileID fid) throws (1: FileDoesNotExists e), + list getQueue(), + list getCollector(), + list getQueueData(), + list getCollectorData(), + map getPackageOrder(1: Destination destination), + map getFileOrder(1: PackageID pid) + + // downloads - adding/deleting + list generateAndAddPackages(1: LinkList links, 2: Destination dest), + PackageID addPackage(1: string name, 2: LinkList links, 3: Destination dest, 4: string password), + void addFiles(1: PackageID pid, 2: LinkList links), + void uploadContainer(1: string filename, 2: binary data), + void deleteFiles(1: list fids), + void deletePackages(1: list pids), + + // downloads - modifying + void pushToQueue(1: PackageID pid), + void pullFromQueue(1: PackageID pid), + void restartPackage(1: PackageID pid), + void restartFile(1: FileID fid), + void recheckPackage(1: PackageID pid), + void stopAllDownloads(), + void stopDownloads(1: list fids), + void setPackageName(1: PackageID pid, 2: string name), + void movePackage(1: Destination destination, 2: PackageID pid), + void moveFiles(1: list fids, 2: PackageID pid), + void orderPackage(1: PackageID pid, 2: i16 position), + void orderFile(1: FileID fid, 2: i16 position), + void setPackageData(1: PackageID pid, 2: map data) throws (1: PackageDoesNotExists e), + list deleteFinished(), + void restartFailed(), + + //events + list getEvents(1: string uuid) + + //accounts + list getAccounts(1: bool refresh), + list getAccountTypes() + void updateAccount(1: PluginName plugin, 2: string account, 3: string password, 4: map options), + void removeAccount(1: PluginName plugin, 2: string account), + + //auth + bool login(1: string username, 2: string password), + UserData getUserData(1: string username, 2:string password) throws (1: UserDoesNotExists ex), + map getAllUserData(), + + //services + + // servicename : description + map> getServices(), + bool hasService(1: PluginName plugin, 2: string func), + string call(1: ServiceCall info) throws (1: ServiceDoesNotExists ex, 2: ServiceException e), + + + //info + // {plugin: {name: value}} + map> getAllInfo(), + map getInfoByPlugin(1: PluginName plugin), + + //scheduler + + // TODO + + + // User interaction + + //captcha + bool isCaptchaWaiting(), + CaptchaTask getCaptchaTask(1: bool exclusive), + string getCaptchaTaskStatus(1: InteractionID tid), + void setCaptchaResult(1: InteractionID tid, 2: string result), + } + .. [[[end]]] + diff --git a/docs/api/overview.rst b/docs/api/overview.rst new file mode 100644 index 000000000..02cee3e0d --- /dev/null +++ b/docs/api/overview.rst @@ -0,0 +1,35 @@ +.. _overview: + +======================================= +API - Application Programming Interface +======================================= + +From Wikipedia, the free encyclopedia [1]_: + + An application programming interface (API) is a source code based specification intended to be used as an interface + by software components to communicate with each other. An API may include specifications for routines, + data structures, object classes, and variables. + +.. rubric:: Motivation + +The idea of the centralized pyLoad :class:`Api ` is to give uniform access to all integral parts +and plugins in pyLoad, and furthermore to other clients, written in arbitrary programming languages. +Most of the :class:`Api ` functionality is exposed via RPC [2]_ and accessible via thrift [3]_ or +simple JSON objects [4]_. In conclusion the :class:`Api ` is accessible via many programming language, +over network from remote maschines and over browser with javascript. + + +.. rubric:: Contents + +.. toctree:: + + access_api.rst + datatypes.rst + + +.. rubric:: Footnotes + +.. [1] http://en.wikipedia.org/wiki/Application_programming_interface +.. [2] http://en.wikipedia.org/wiki/Remote_procedure_call +.. [3] http://en.wikipedia.org/wiki/Thrift_(protocol) +.. [4] http://en.wikipedia.org/wiki/Json \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 09e4d0c1c..454ed5967 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -65,8 +65,8 @@ v = options.version.split(".") cog.outl("version = '%s'" % ".".join(v[:2])) cog.outl("release = '%s'" % ".".join(v)) ]]]""" -version = '0.4' -release = '0.4.9' +version = '0.5' +release = '0.5.0' # [[[end]]] diff --git a/docs/extend_pyload.rst b/docs/extend_pyload.rst deleted file mode 100755 index 337cb6854..000000000 --- a/docs/extend_pyload.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _extend_pyload: - -******************** -How to extend pyLoad -******************** - -In general there a two different plugin types. These allow everybody to write powerful, modular plugins without knowing -every detail of the pyLoad core. However you should have some basic knowledge of python. - -.. toctree:: - - write_hooks.rst - write_plugins.rst \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 757fd7537..befac0fd2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,27 +1,46 @@ -.. pyLoad documentation master file, created by - sphinx-quickstart on Sat Jun 4 11:54:34 2011. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. pyLoad documentation master file -Welcome to pyLoad's documentation! -================================== +===================== +pyLoad Documentation +===================== -Great that you found your way to the pyLoad documentation! +.. image:: _static/logo.png + :height: 144 + :width: 412 -We have collected some information here to help developer writing plugins and understandig our code. -If you want to help us developing visit us in our IRC channel #pyload on freenode.net or leave a message in our forum. -Contents: +Great that you found your way to the pyLoad [1]_ documentation! + +This is the ultimate document to get started extending or accessing pyLoad in your own way. +We will cover on how to access the API so you can write your own client to pyLoad. The next big part gives you an idea +how to extend pyLoad and write your own powerful plugins, which perfectly integrate into our system. + +The complete pyLoad source and this documentation is available at bitbucket [2]_. If you would like to contribute +come around in our irc channel [3]_ or open a pull request. +In case you still have question, ask them at our forum [4]_ or in our official irc channel at #pyload @ irc.freenode.net + +We wish you the best of luck and happy programming. +-- the pyLoad Team + +Contents +-------- .. toctree:: :maxdepth: 2 - access_api.rst - extend_pyload.rst + api/overview.rst + plugins/overview.rst module_overview.rst .. currentmodule:: module +.. rubric:: Footnotes + +.. [1] http://pyload.org +.. [2] http://pyload.org/irc +.. [3] http://bitbucket.org/spoob/pyload/overview +.. [4] http://forum.pyload.org + ================== * :ref:`genindex` diff --git a/docs/module_overview.rst b/docs/module_overview.rst index 309dccdfd..b8db51538 100644 --- a/docs/module_overview.rst +++ b/docs/module_overview.rst @@ -1,7 +1,8 @@ + Module Overview =============== -You can find an overview of some important classes here: +A little selection of most important modules in pyLoad. .. autosummary:: :toctree: module @@ -10,8 +11,9 @@ You can find an overview of some important classes here: module.plugins.Base.Base module.plugins.Hoster.Hoster module.plugins.Crypter.Crypter - module.plugins.Account.Account module.plugins.Hook.Hook + module.plugins.Account.Account + module.plugins.MultiHoster.MultiHoster module.HookManager.HookManager module.interaction.EventManager.EventManager module.interaction.InteractionManager.InteractionManager diff --git a/docs/plugins/account_plugin.rst b/docs/plugins/account_plugin.rst new file mode 100644 index 000000000..75bf61a75 --- /dev/null +++ b/docs/plugins/account_plugin.rst @@ -0,0 +1,5 @@ +.. _account_plugin: + +Account - Premium Access +======================== + diff --git a/docs/plugins/base_plugin.rst b/docs/plugins/base_plugin.rst new file mode 100644 index 000000000..62ab248ef --- /dev/null +++ b/docs/plugins/base_plugin.rst @@ -0,0 +1,5 @@ +.. _base_plugin: + +Base Plugin - And here it begins... +=================================== + diff --git a/docs/plugins/crypter_plugin.rst b/docs/plugins/crypter_plugin.rst new file mode 100644 index 000000000..d910ec412 --- /dev/null +++ b/docs/plugins/crypter_plugin.rst @@ -0,0 +1,5 @@ +.. _crypter_plugin: + +Crypter - Extract links from pages +================================== + diff --git a/docs/plugins/hook_plugin.rst b/docs/plugins/hook_plugin.rst new file mode 100644 index 000000000..e263ece2e --- /dev/null +++ b/docs/plugins/hook_plugin.rst @@ -0,0 +1,162 @@ +.. _write_hooks: + +Hook - Do everything you want +============================= + +A Hook is a python file which is located at :file:`module/plugins/hooks`. +The :class:`HookManager ` will load it automatically on startup. Only one instance exists +over the complete lifetime of pyload. Your hook can interact on various events called by the :class:`HookManager `, +do something complete autonomic and has full access to the :class:`Api ` and every detail of pyLoad. +The UpdateManager, CaptchaTrader, UnRar and many more are realised as hooks. + +Hook header +----------- + +Your hook needs to subclass :class:`Hook ` and will inherit all of its method, make sure to check its documentation! + +All Hooks should start with something like this: :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + __name__ = "YourHook" + __version__ = "0.1" + __description__ = "Does really cool stuff" + __config__ = [ ("activated" , "bool" , "Activated" , "True" ) ] + __threaded__ = ["downloadFinished"] + __author_name__ = ("Me") + __author_mail__ = ("me@has-no-mail.com") + +All meta-data is defined in the header, you need at least one option at ``__config__`` so the user can toggle your +hook on and off. Dont't overwrite the ``init`` method if not neccesary, use ``setup`` instead. + +Using the Config +---------------- + +We are taking a closer look at the ``__config__`` parameter. +You can add more config values as desired by adding tuples of the following format to the config list: ``("name", "type", "description", "default value")``. +When everything went right you can access the config values with ``self.getConfig(name)`` and ``self.setConfig(name,value``. + + +Interacting on Events +--------------------- + +The next step is to think about where your Hook action takes places. + +The easiest way is to overwrite specific methods defined by the :class:`Hook ` base class. +The name is indicating when the function gets called. +See :class:`Hook ` page for a complete listing. + +You should be aware of the arguments the Hooks are called with, whether its a :class:`PyFile ` +or :class:`PyPackage ` you should read its related documentation to know how to access her great power and manipulate them. + +A basic excerpt would look like: :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + + def coreReady(self): + print "Yay, the core is ready let's do some work." + + def downloadFinished(self, pyfile): + print "A Download just finished." + +Another important feature to mention can be seen at the ``__threaded__`` parameter. Function names listed will be executed +in a thread, in order to not block the main thread. This should be used for all kind of longer processing tasks. + +Another and more flexible and powerful way is to use event listener. +All hook methods exists as event and very useful additional events are dispatched by the core. For a little overview look +at :class:`HookManager `. Keep in mind that you can define own events and other people may listen on them. + +For your convenience it's possible to register listeners automatical via the ``event_map`` attribute. +It requires a `dict` that maps event names to function names or a `list` of function names. It's important that all names are strings :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + event_map = {"downloadFinished" : "doSomeWork", + "allDownloadsFnished": "someMethod", + "coreReady": "initialize"} + + def initialize(self): + print "Initialized." + + def doSomeWork(self, pyfile): + print "This is equivalent to the above example." + + def someMethod(self): + print "The underlying event (allDownloadsFinished) for this method is not available through the base class" + +An advantage of the event listener is that you are able to register and remove the listeners at runtime. +Use `self.manager.addEvent("name", function)`, `self.manager.removeEvent("name", function)` and see doc for +:class:`HookManager `. Contrary to ``event_map``, ``function`` has to be a reference +and **not** a `string`. + +We introduced events because it scales better if there a a huge amount of events and hooks. So all future interaction will be exclusive +available as event and not accessible through overwriting hook methods. However you can safely do this, it will not be removed and is easier to implement. + + +Providing RPC services +---------------------- + +You may noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. +So probably clients want to be able to interact with your hook to request it's state or invoke some action. + +Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: + + from module.plugins.Hook import Hook, Expose + + class YourHook(Hook): + """ + Your Hook code here. + """ + + @Expose + def invoke(self, arg): + print "Invoked with", arg + +Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. +Here is a basic example: :: + + #Assuming client is a ThriftClient or Api object + + print client.getServices() + print client.call(ServiceCall("YourHook", "invoke", "an argument")) + +Providing status information +---------------------------- +Your hook can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. + +Just store everything in ``self.info``. :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + + def setup(self): + self.info = {"running": False} + + def coreReady(self): + self.info["running"] = True + +Usable with: :: + + #Assuming client is a ThriftClient or Api object + + print client.getAllInfo() + +Example +------- + Sorry but you won't find an example here ;-) + + Look at :file:`module/plugins/hooks` and you will find plenty examples there. diff --git a/docs/plugins/hoster_plugin.rst b/docs/plugins/hoster_plugin.rst new file mode 100644 index 000000000..59f35f5cb --- /dev/null +++ b/docs/plugins/hoster_plugin.rst @@ -0,0 +1,102 @@ +.. _hoster_plugin: + +Hoster - Load files to disk +=========================== + +A Plugin is a python file located at one of the subfolders in :file:`module/plugins/`. Either :file:`hoster`, :file:`crypter` +or :file:`container`, depending of it's type. + +There are three kinds of different plugins: **Hoster**, **Crypter**, **Container**. +All kind of plugins inherit from the base :class:`Plugin `. You should know its +convenient methods, they make your work easier ;-) + +Every plugin defines a ``__pattern__`` and when the user adds urls, every url is matched against the pattern defined in +the plugin. In case the ``__pattern__`` matched on the url the plugin will be assigned to handle it and instanciated when +pyLoad begins to download/decrypt the url. + +Plugin header +------------- + +How basic hoster plugin header could look like: :: + + from module.plugin.Hoster import Hoster + + class MyFileHoster(Hoster): + __version__ = "0.1" + __pattern__ = r"http://myfilehoster.example.com/file_id/[0-9]+" + __config__ = [] + +You have to define these meta-data, ``__pattern__`` has to be a regexp that sucessfully compiles with +``re.compile(__pattern__)``. + +Just like :ref:`write_hooks` you can add and use config values exatly the same way. +If you want a Crypter or Container plugin, just replace the word Hoster with your desired plugin type. + + +Hoster plugins +-------------- + +We head to the next important section, the ``process`` method of your plugin. +In fact the ``process`` method is the only functionality your plugin has to provide, but its always a good idea to split up tasks to not produce spaghetti code. +An example ``process`` function could look like this :: + + from module.plugin.Hoster import Hoster + + class MyFileHoster(Hoster): + """ + plugin code + """ + + def process(self, pyfile): + html = self.load(pyfile.url) # load the content of the orginal pyfile.url to html + + # parse the name from the site and set attribute in pyfile + pyfile.name = self.myFunctionToParseTheName(html) + parsed_url = self.myFunctionToParseUrl(html) + + # download the file, destination is determined by pyLoad + self.download(parsed_url) + +You need to know about the :class:`PyFile ` class, since an instance of it is given as parameter to every pyfile. +Some tasks your plugin should handle: proof if file is online, get filename, wait if needed, download the file, etc.. + +Wait times +__________ + +Some hoster require you to wait a specific time. Just set the time with ``self.setWait(seconds)`` or +``self.setWait(seconds, True)`` if you want pyLoad to perform a reconnect if needed. + +Captcha decrypting +__________________ + +To handle captcha input just use ``self.decryptCaptcha(url, ...)``, it will be send to clients +or handled by :class:`Hook ` plugins + +Crypter +------- + +What about Decrypter and Container plugins? +Well, they work nearly the same, only that the function they have to provide is named ``decrypt`` + +Example: :: + + from module.plugin.Crypter import Crypter + + class MyFileCrypter(Crypter): + """ + plugin code + """ + def decrypt(self, pyfile): + + urls = ["http://get.pyload.org/src", "http://get.pyload.org/debian", "http://get.pyload.org/win"] + + self.packages.append(("pyLoad packages", urls, "pyLoad packages")) # urls list of urls + +They can access all the methods from :class:`Plugin `, but the important thing is they +have to append all packages they parsed to the `self.packages` list. Simply append tuples with `(name, urls, folder)`, +where urls is the list of urls contained in the packages. Thats all of your work, pyLoad will know what to do with them. + +Examples +-------- + +Best examples are already existing plugins in :file:`module/plugins/`. \ No newline at end of file diff --git a/docs/plugins/overview.rst b/docs/plugins/overview.rst new file mode 100755 index 000000000..23913b787 --- /dev/null +++ b/docs/plugins/overview.rst @@ -0,0 +1,31 @@ +.. _overview: + +================ +Extending pyLoad +================ + +.. pull-quote:: + Any sufficiently advanced technology is indistinguishable from magic. + + -- Arthur C. Clarke + + +.. rubric:: Motivation + +pyLoad offers an comfortable and powerful plugin system to make extending possible. With it you only need to have some +python knowledge and can just start right away writing your own plugins. This document gives you an overwiew about the +conceptual part. You should not left out the `Base` part, since it contains basic functionality for all plugins types. + +.. rubric:: Contents + +.. toctree:: + + base_plugin.rst + crypter_plugin.rst + hoster_plugin.rst + account_plugin.rst + hook_plugin.rst + + + +.. rubric:: Footnotes \ No newline at end of file diff --git a/docs/write_hooks.rst b/docs/write_hooks.rst deleted file mode 100644 index dd60367b7..000000000 --- a/docs/write_hooks.rst +++ /dev/null @@ -1,162 +0,0 @@ -.. _write_hooks: - -Hooks -===== - -A Hook is a python file which is located at :file:`module/plugins/hooks`. -The :class:`HookManager ` will load it automatically on startup. Only one instance exists -over the complete lifetime of pyload. Your hook can interact on various events called by the :class:`HookManager `, -do something complete autonomic and has full access to the :class:`Api ` and every detail of pyLoad. -The UpdateManager, CaptchaTrader, UnRar and many more are realised as hooks. - -Hook header ------------ - -Your hook needs to subclass :class:`Hook ` and will inherit all of its method, make sure to check its documentation! - -All Hooks should start with something like this: :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - __name__ = "YourHook" - __version__ = "0.1" - __description__ = "Does really cool stuff" - __config__ = [ ("activated" , "bool" , "Activated" , "True" ) ] - __threaded__ = ["downloadFinished"] - __author_name__ = ("Me") - __author_mail__ = ("me@has-no-mail.com") - -All meta-data is defined in the header, you need at least one option at ``__config__`` so the user can toggle your -hook on and off. Dont't overwrite the ``init`` method if not neccesary, use ``setup`` instead. - -Using the Config ----------------- - -We are taking a closer look at the ``__config__`` parameter. -You can add more config values as desired by adding tuples of the following format to the config list: ``("name", "type", "description", "default value")``. -When everything went right you can access the config values with ``self.getConfig(name)`` and ``self.setConfig(name,value``. - - -Interacting on Events ---------------------- - -The next step is to think about where your Hook action takes places. - -The easiest way is to overwrite specific methods defined by the :class:`Hook ` base class. -The name is indicating when the function gets called. -See :class:`Hook ` page for a complete listing. - -You should be aware of the arguments the Hooks are called with, whether its a :class:`PyFile ` -or :class:`PyPackage ` you should read its related documentation to know how to access her great power and manipulate them. - -A basic excerpt would look like: :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - """ - Your Hook code here. - """ - - def coreReady(self): - print "Yay, the core is ready let's do some work." - - def downloadFinished(self, pyfile): - print "A Download just finished." - -Another important feature to mention can be seen at the ``__threaded__`` parameter. Function names listed will be executed -in a thread, in order to not block the main thread. This should be used for all kind of longer processing tasks. - -Another and more flexible and powerful way is to use event listener. -All hook methods exists as event and very useful additional events are dispatched by the core. For a little overview look -at :class:`HookManager `. Keep in mind that you can define own events and other people may listen on them. - -For your convenience it's possible to register listeners automatical via the ``event_map`` attribute. -It requires a `dict` that maps event names to function names or a `list` of function names. It's important that all names are strings :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - """ - Your Hook code here. - """ - event_map = {"downloadFinished" : "doSomeWork", - "allDownloadsFnished": "someMethod", - "coreReady": "initialize"} - - def initialize(self): - print "Initialized." - - def doSomeWork(self, pyfile): - print "This is equivalent to the above example." - - def someMethod(self): - print "The underlying event (allDownloadsFinished) for this method is not available through the base class" - -An advantage of the event listener is that you are able to register and remove the listeners at runtime. -Use `self.manager.addEvent("name", function)`, `self.manager.removeEvent("name", function)` and see doc for -:class:`HookManager `. Contrary to ``event_map``, ``function`` has to be a reference -and **not** a `string`. - -We introduced events because it scales better if there a a huge amount of events and hooks. So all future interaction will be exclusive -available as event and not accessible through overwriting hook methods. However you can safely do this, it will not be removed and is easier to implement. - - -Providing RPC services ----------------------- - -You may noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. -So probably clients want to be able to interact with your hook to request it's state or invoke some action. - -Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: - - from module.plugins.Hook import Hook, Expose - - class YourHook(Hook): - """ - Your Hook code here. - """ - - @Expose - def invoke(self, arg): - print "Invoked with", arg - -Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. -Here is a basic example: :: - - #Assuming client is a ThriftClient or Api object - - print client.getServices() - print client.call(ServiceCall("YourHook", "invoke", "an argument")) - -Providing status information ----------------------------- -Your hook can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. - -Just store everything in ``self.info``. :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - """ - Your Hook code here. - """ - - def setup(self): - self.info = {"running": False} - - def coreReady(self): - self.info["running"] = True - -Usable with: :: - - #Assuming client is a ThriftClient or Api object - - print client.getAllInfo() - -Example -------- - Sorry but you won't find an example here ;-) - - Look at :file:`module/plugins/hooks` and you will find plenty examples there. diff --git a/docs/write_plugins.rst b/docs/write_plugins.rst deleted file mode 100644 index b513a5978..000000000 --- a/docs/write_plugins.rst +++ /dev/null @@ -1,103 +0,0 @@ -.. _write_plugins: - -Plugins -======= - -A Plugin is a python file located at one of the subfolders in :file:`module/plugins/`. Either :file:`hoster`, :file:`crypter` -or :file:`container`, depending of it's type. - -There are three kinds of different plugins: **Hoster**, **Crypter**, **Container**. -All kind of plugins inherit from the base :class:`Plugin `. You should know its -convenient methods, they make your work easier ;-) - -Every plugin defines a ``__pattern__`` and when the user adds urls, every url is matched against the pattern defined in -the plugin. In case the ``__pattern__`` matched on the url the plugin will be assigned to handle it and instanciated when -pyLoad begins to download/decrypt the url. - -Plugin header -------------- - -How basic hoster plugin header could look like: :: - - from module.plugin.Hoster import Hoster - - class MyFileHoster(Hoster): - __name__ = "MyFileHoster" - __version__ = "0.1" - __pattern__ = r"http://myfilehoster.example.com/file_id/[0-9]+" - __config__ = [] - -You have to define these meta-data, ``__pattern__`` has to be a regexp that sucessfully compiles with -``re.compile(__pattern__)``. - -Just like :ref:`write_hooks` you can add and use config values exatly the same way. -If you want a Crypter or Container plugin, just replace the word Hoster with your desired plugin type. - - -Hoster plugins --------------- - -We head to the next important section, the ``process`` method of your plugin. -In fact the ``process`` method is the only functionality your plugin has to provide, but its always a good idea to split up tasks to not produce spaghetti code. -An example ``process`` function could look like this :: - - from module.plugin.Hoster import Hoster - - class MyFileHoster(Hoster): - """ - plugin code - """ - - def process(self, pyfile): - html = self.load(pyfile.url) # load the content of the orginal pyfile.url to html - - # parse the name from the site and set attribute in pyfile - pyfile.name = self.myFunctionToParseTheName(html) - parsed_url = self.myFunctionToParseUrl(html) - - # download the file, destination is determined by pyLoad - self.download(parsed_url) - -You need to know about the :class:`PyFile ` class, since an instance of it is given as parameter to every pyfile. -Some tasks your plugin should handle: proof if file is online, get filename, wait if needed, download the file, etc.. - -Wait times -__________ - -Some hoster require you to wait a specific time. Just set the time with ``self.setWait(seconds)`` or -``self.setWait(seconds, True)`` if you want pyLoad to perform a reconnect if needed. - -Captcha decrypting -__________________ - -To handle captcha input just use ``self.decryptCaptcha(url, ...)``, it will be send to clients -or handled by :class:`Hook ` plugins - -Crypter -------- - -What about Decrypter and Container plugins? -Well, they work nearly the same, only that the function they have to provide is named ``decrypt`` - -Example: :: - - from module.plugin.Crypter import Crypter - - class MyFileCrypter(Crypter): - """ - plugin code - """ - def decrypt(self, pyfile): - - urls = ["http://get.pyload.org/src", "http://get.pyload.org/debian", "http://get.pyload.org/win"] - - self.packages.append(("pyLoad packages", urls, "pyLoad packages")) # urls list of urls - -They can access all the methods from :class:`Plugin `, but the important thing is they -have to append all packages they parsed to the `self.packages` list. Simply append tuples with `(name, urls, folder)`, -where urls is the list of urls contained in the packages. Thats all of your work, pyLoad will know what to do with them. - -Examples --------- - -Best examples are already existing plugins in :file:`module/plugins/`. \ No newline at end of file -- cgit v1.2.3 From 6eaa7bb25e2254c80c43fe46166142d590e86c64 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 7 Jan 2012 23:58:28 +0100 Subject: some cleanups --- docs/_static/logo.png | Bin 0 -> 29742 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/_static/logo.png (limited to 'docs') diff --git a/docs/_static/logo.png b/docs/_static/logo.png new file mode 100644 index 000000000..1a11f5cc0 Binary files /dev/null and b/docs/_static/logo.png differ -- cgit v1.2.3 From 9ebdb33dcd859e02e02aff628922047c865e7124 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 9 Jan 2012 21:29:18 +0100 Subject: updated docs --- docs/api/access_api.rst | 122 ------------------------------------------- docs/api/json_api.rst | 72 +++++++++++++++++++++++++ docs/api/overview.rst | 5 +- docs/api/thrift_api.rst | 74 ++++++++++++++++++++++++++ docs/conf.py | 11 ++-- docs/index.rst | 9 ++-- docs/plugins/hook_plugin.rst | 2 +- 7 files changed, 161 insertions(+), 134 deletions(-) delete mode 100644 docs/api/access_api.rst create mode 100644 docs/api/json_api.rst create mode 100644 docs/api/thrift_api.rst (limited to 'docs') diff --git a/docs/api/access_api.rst b/docs/api/access_api.rst deleted file mode 100644 index efa1ae3fc..000000000 --- a/docs/api/access_api.rst +++ /dev/null @@ -1,122 +0,0 @@ -.. _access_api: - -********************* -How to access the API -********************* - -pyLoad has a very powerfull API with can be accessed in several ways. - -Overview --------- - -First of all, you need to know what you can do with our API. It lets you do all common task like -retrieving download status, manage queue, manage accounts, modify config and so on. - -This document is not intended to explain every function in detail, for a complete listing -see :class:`Api `. - -Of course its possible to access the ``core.api`` attribute in plugins and hooks, but much more -interesting is the possibillity to call function from different programs written in many different languages. - -pyLoad uses thrift as backend and provides its :class:`Api ` as service. -More information about thrift can be found here http://wiki.apache.org/thrift/. - - -Using Thrift ------------- - -Every thrift service has to define all data structures and declare every method which should be usable via rpc. -This file is located at :file:`module/remote/thriftbackend/pyload.thrift`, its very helpful to inform about -arguments and detailed structure of return types. However it does not contain any information about what the functions does. -You can also look at it :doc:`here ` - -Assuming you want to use the API in any other language than python than check if it is -supported here http://wiki.apache.org/thrift/LibraryFeatures?action=show&redirect=LanguageSupport. - -Now install thrift, for instructions see http://wiki.apache.org/thrift/ThriftInstallation. -If every thing went fine you are ready to generate the method stubs, the command basically looks like this. :: - - $ thrift --gen (language) pyload.thrift - -You find now a directory named :file:`gen-(language)`. For instruction how to use the generated files consider the docs -at the thrift wiki and the examples here http://wiki.apache.org/thrift/ThriftUsage. - - -======= -Example -======= -In case you want to use python, pyload has already all files included to access the api over rpc. - -A basic script that prints out some information: :: - - from module.remote.thriftbackend.ThriftClient import ThriftClient, WrongLogin - - try: - client = ThriftClient(host="127.0.0.1", port=7227, user="User", password="yourpw") - except: - print "Login was wrong" - exit() - - print "Server version:", client.getServerVersion() - print client.statusDownloads() - q = client.getQueue() - for p in q: - data = client.getPackageData(p.pid) - print "Package Name: ", data.name - -That's all for now, pretty easy isn't it? -If you still have open questions come around in irc or post them at our pyload forum. - - -Using HTTP/JSON ---------------- - -Another maybe easier way, which does not require much setup is to access the JSON Api via HTTP. -For this reason the webinterface must be enabled. - -===== -Login -===== - -First you need to authenticate, if you using this within the webinterface and the user is logged the API is also accessible, -since they share the same cookie/session. - -However, if you are building a external client and want to authenticate manually -you have to send your credentials ``username`` and ``password`` as -POST parameter to ``http://pyload-core/api/login``. - -The result will be your session id. If you are using cookies, it will be set and you can use the API now. -In case you dont't have cookies enabled you can pass the session id as ``session`` POST parameter -so pyLoad can authenticate you. - -=============== -Calling Methods -=============== - -In general you can use any method listed at the :class:`Api ` documentation, which is also available to -the thriftbackend. - -Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address -or hostname including the webinterface port. By default on local access this would be `localhost:8000`. - -The return value will be formatted in JSON, complex data types as dictionaries. -As mentionted above for a documentation about the return types look at the thrift specification file :file:`module/remote/thriftbackend/pyload.thrift`. - -================== -Passing parameters -================== - -To pass arguments you have two choices. -Either use positional arguments, eg ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword arguments -supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names in the :class:`Api ` -documentation. - -It is important that *all* arguments are in JSON format. So ``http://pyload-core/api/getFileData/1`` is valid because -1 represents an integer in json format. On the other hand if the method is expecting strings, this would be correct: -``http://pyload-core/api/getUserData/"username"/"password"``. - -Strings are wrapped in double qoutes, because `"username"` represents a string in json format. It's not limited to strings and intergers, -every container type like lists and dicts are possible. You usually don't have to convert them. just use a json encoder before using them -in the HTTP request. - -Please note that the data have to be urlencoded at last. (Most libaries will do that automatically) \ No newline at end of file diff --git a/docs/api/json_api.rst b/docs/api/json_api.rst new file mode 100644 index 000000000..3df006c49 --- /dev/null +++ b/docs/api/json_api.rst @@ -0,0 +1,72 @@ +.. _json_api: + +======== +JSON API +======== + +JSON [1]_ is a lightweight object notation and wrapper exists for nearly every programming language. Every +modern browser is able to load JSON objects with JavaScript. Unlike to thrift you don't need to generate or precompile +any stub methods, the JSON :class:`Api ` is ready to use for most language. The libary is really lightweight (at least in python) +and you can build very lightweight scripts with it. Because of the builtin support JSON is the first choice for all browser +applications. + +In our case JSON is just the output format, you have exactly the same methods available as with the thrift backend. The only +difference is the used protocol. + +So are there still reasons to choose the original :doc:`thrift ` backend in favor to JSON? Yes, since it +uses a binary protocol the performance will be better (when generating the objects), traffic will be smaller and +therefore the transfer faster. +In most IDEs you will get code completion, because of the pre-generated classes, which can make work much easier. + +If you intend to write a full client you should prefer thrift if the language is supported, for lightweight scripts and +in browser environment JSON wil be the better choice. + +Login +----- + +First you need to authenticate, if you using this within the webinterface and the user is logged the API is also accessible, +since they share the same cookie/session. + +However, if you are building a external client and want to authenticate manually +you have to send your credentials ``username`` and ``password`` as +POST parameter to ``http://pyload-core/api/login``. + +The result will be your session id. If you are using cookies, it will be set and you can use the API now. +In case you dont't have cookies enabled you can pass the session id as ``session`` POST parameter +so pyLoad can authenticate you. + + +Calling Methods +--------------- + +In general you can use any method listed at the :class:`Api ` documentation, which is also available to +the thriftbackend. + +Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address +or hostname including the webinterface port. By default on local access this would be `localhost:8000`. + +The return value will be formatted in JSON, complex data types as dictionaries. Definition for datatypes can be found +:doc:`here ` + +Passing parameters +------------------ + +To pass arguments you have two choices. +Either use positional arguments, eg ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword +arguments supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names +in the :class:`Api ` documentation. + +It is important that *all* arguments are in JSON format. So ``http://pyload-core/api/getFileData/1`` is valid because +1 represents an integer in json format. On the other hand if the method is expecting strings, this would be correct: +``http://pyload-core/api/getUserData/"username"/"password"``. + +Strings are wrapped in double qoutes, because `"username"` represents a string in json format. It's not limited to +strings and intergers, every container type like lists and dicts are possible. You usually don't have to convert them. +Just use a json encoder before using them in the HTTP request. + +Please note that the data have to be urlencoded at last. (Most libaries will do that automatically) + + +.. rubric:: Footnotes + +.. [1] http://de.wikipedia.org/wiki/JavaScript_Object_Notation \ No newline at end of file diff --git a/docs/api/overview.rst b/docs/api/overview.rst index 02cee3e0d..47fe1be82 100644 --- a/docs/api/overview.rst +++ b/docs/api/overview.rst @@ -23,7 +23,8 @@ over network from remote maschines and over browser with javascript. .. toctree:: - access_api.rst + thrift_api.rst + json_api.rst datatypes.rst @@ -31,5 +32,5 @@ over network from remote maschines and over browser with javascript. .. [1] http://en.wikipedia.org/wiki/Application_programming_interface .. [2] http://en.wikipedia.org/wiki/Remote_procedure_call -.. [3] http://en.wikipedia.org/wiki/Thrift_(protocol) +.. [3] ``_ .. [4] http://en.wikipedia.org/wiki/Json \ No newline at end of file diff --git a/docs/api/thrift_api.rst b/docs/api/thrift_api.rst new file mode 100644 index 000000000..a4987a797 --- /dev/null +++ b/docs/api/thrift_api.rst @@ -0,0 +1,74 @@ +.. _thrift_api: + +========== +Thrift API +========== + +Thrift [1]_ was first developed in-house at facebook, but later published to public domain and developed at Apache Incubator. +It includes a binary protocol for remote calls, which is much more performant than other data formats like XML, additionally +it is available for numerous languages and therefore we choosed it as primary backend for our API. + +First of all, you need to know what you can do with our API. It lets you do all common task like +retrieving download status, manage queue, manage accounts, modify config and so on. + +This document is not intended to explain every function in detail, for a complete listing +see :class:`Api `. + +Of course its possible to access the ``core.api`` attribute in plugins and hooks, but much more +interesting is the possibillity to call function from different programs written in many different languages. + +pyLoad uses thrift as backend and provides its :class:`Api ` as service. +More information about thrift can be found in their wiki [2]_. + + +Using Thrift +------------ + +Every thrift service has to define all data structures and declare every method which should be usable via rpc. +This file is located at :file:`module/remote/thriftbackend/pyload.thrift`, its very helpful to inform about +arguments and detailed structure of return types. However it does not contain any information about what the functions does. +You can also look at it :doc:`here ` + +Assuming you want to use the API in any other language than python than check if it is supported [3]_. + +Now install thrift, for instructions see [4]_. +If every thing went fine you are ready to generate the method stubs, the command basically looks like this. :: + + $ thrift --gen (language) pyload.thrift + +You find now a directory named :file:`gen-(language)`. For instruction how to use the generated files consider the docs +at the thrift wiki, as well at the examples [5]_. + + +Example +------- + +In case you want to use python, pyload has already all files included to access the api over rpc. + +A basic script that prints out some information: :: + + from module.remote.thriftbackend.ThriftClient import ThriftClient, WrongLogin + + try: + client = ThriftClient(host="127.0.0.1", port=7227, user="User", password="yourpw") + except: + print "Login was wrong" + exit() + + print "Server version:", client.getServerVersion() + print client.statusDownloads() + q = client.getQueue() + for p in q: + data = client.getPackageData(p.pid) + print "Package Name: ", data.name + +That's all for now, pretty easy isn't it? +If you still have open questions come around in irc or post them at our pyload forum. + +.. rubric:: Footnotes + +.. [1] http://en.wikipedia.org/wiki/Thrift_(protocol) +.. [2] http://wiki.apache.org/thrift/ +.. [3] http://wiki.apache.org/thrift/LibraryFeatures?action=show&redirect=LanguageSupport +.. [4] http://wiki.apache.org/thrift/ThriftInstallation +.. [5] http://wiki.apache.org/thrift/ThriftUsage \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 454ed5967..4961fc910 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,11 +27,12 @@ sys.path.append(join(dir_name, "module", "lib")) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', + 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] autosummary_generate = True autodoc_default_flags = ['members'] @@ -196,8 +197,8 @@ htmlhelp_basename = 'pyLoaddoc' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'pyLoad.tex', u'pyLoad Documentation', - u'pyLoad Team', 'manual'), + ('index', 'pyLoad.tex', u'pyLoad Documentation', + u'pyLoad Team', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -235,4 +236,4 @@ man_pages = [ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = {'http://docs.python.org/': None} \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index befac0fd2..31d688e65 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,14 +12,15 @@ pyLoad Documentation Great that you found your way to the pyLoad [1]_ documentation! This is the ultimate document to get started extending or accessing pyLoad in your own way. -We will cover on how to access the API so you can write your own client to pyLoad. The next big part gives you an idea -how to extend pyLoad and write your own powerful plugins, which perfectly integrate into our system. +We will cover on how to access the API so you can write your own client to pyLoad. In the next step you will be given +an idea on how to extend pyLoad and write your own powerful plugins, which perfectly integrate into our system. The complete pyLoad source and this documentation is available at bitbucket [2]_. If you would like to contribute come around in our irc channel [3]_ or open a pull request. -In case you still have question, ask them at our forum [4]_ or in our official irc channel at #pyload @ irc.freenode.net +In case you still have questions, ask at our forum [4]_ or in our official irc channel #pyload @ irc.freenode.net + +We wish you happy programming! -We wish you the best of luck and happy programming. -- the pyLoad Team Contents diff --git a/docs/plugins/hook_plugin.rst b/docs/plugins/hook_plugin.rst index e263ece2e..be1097057 100644 --- a/docs/plugins/hook_plugin.rst +++ b/docs/plugins/hook_plugin.rst @@ -1,6 +1,6 @@ .. _write_hooks: -Hook - Do everything you want +Hook - Do whatever you want ============================= A Hook is a python file which is located at :file:`module/plugins/hooks`. -- cgit v1.2.3 From c654f31efc548957f10e3f4c1a5060dcea1dcdec Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 10 Jan 2012 00:04:24 +0100 Subject: changed HEAD request --- docs/plugins/base_plugin.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'docs') diff --git a/docs/plugins/base_plugin.rst b/docs/plugins/base_plugin.rst index 62ab248ef..4ffe2e457 100644 --- a/docs/plugins/base_plugin.rst +++ b/docs/plugins/base_plugin.rst @@ -3,3 +3,22 @@ Base Plugin - And here it begins... =================================== + +Meta Data +--------- + + +Config Entries +-------------- + + +Tagging Guidelines +------------------ + + +Basic Methods +------------- + +Debugging +--------- + -- cgit v1.2.3 From 180a9ee57a6f4eaa5f4bdd7a272057231f6a5c88 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 29 Jan 2012 20:09:11 +0100 Subject: doc for base plugin --- docs/api/datatypes.rst | 2 +- docs/plugins/base_plugin.rst | 84 +++++++++++++++++++++++++++++++++++++++++ docs/plugins/crypter_plugin.rst | 20 ++++++++++ docs/plugins/hoster_plugin.rst | 62 +++--------------------------- docs/plugins/overview.rst | 2 +- 5 files changed, 112 insertions(+), 58 deletions(-) (limited to 'docs') diff --git a/docs/api/datatypes.rst b/docs/api/datatypes.rst index 02e3209cc..ad8c3df80 100644 --- a/docs/api/datatypes.rst +++ b/docs/api/datatypes.rst @@ -286,8 +286,8 @@ for various languages. It is also a good overview of avaible methods and return // downloads - adding/deleting list generateAndAddPackages(1: LinkList links, 2: Destination dest), PackageID addPackage(1: string name, 2: LinkList links, 3: Destination dest, 4: string password), + PackageID uploadContainer(1: string filename, 2: binary data), void addFiles(1: PackageID pid, 2: LinkList links), - void uploadContainer(1: string filename, 2: binary data), void deleteFiles(1: list fids), void deletePackages(1: list pids), diff --git a/docs/plugins/base_plugin.rst b/docs/plugins/base_plugin.rst index 4ffe2e457..e3c72994b 100644 --- a/docs/plugins/base_plugin.rst +++ b/docs/plugins/base_plugin.rst @@ -3,22 +3,106 @@ Base Plugin - And here it begins... =================================== +A Plugin in pyLoad is a python file located at one of the subfolders in :file:`module/plugins/`. +All different plugin types inherit from :class:`Base `, which defines basic methods +and meta data. You should read this section carefully, because it's the base for all plugin development. +After that it is a good idea to look at several already existing plugin to get a more detailed idea of how +they have to look like and whats possible with them. Meta Data --------- +All important data which must be known by pyLoad is set using class attributes pre- and suffixed with ``__``. +An overview of acceptible values can be found in :class:`Base ` source code. +Non needed attributes can be left out, except ``__version__``. Nevertheless please fill out most information +as you can, when you want to submit your plugin to the public repo. + +You don't need to subclass :class:`Base ` directly, but the +intermediate type according to your plugin. As example we choose an Hoster plugin, but the same is true for all +plugin types. + +For localization pyLoad supports gettext [1]_, to mark strings for translation surround them with ``_("...")``. + +How basic hoster plugin header could look like:: + + from module.plugin.Hoster import Hoster + + class MyFileHoster(Hoster): + __version__ = "0.1" + __description__ = _("Short description of the plugin") + __long_description = _("""A even longer description + is not needed for hoster plugin, + but hook plugin should have it so the user knows what they doing.""") Config Entries -------------- +Every plugin is allowed to add entries to the config. These are defined via ``__config__`` and consists +of a list with tuples in the format of ``(name, type, verbose_name, default_value)`` or +``(name, type, verbose_name, short_description, default_value)``. + +Example from Youtube plugin:: + + class YoutubeCom: + __config__ = [("quality", "sd;hd;fullhd", _("Quality Setting"), "hd"), + ("fmt", "int", _("FMT Number 0-45"), _("Desired FMT number, look them up at wikipedia"), 0), + (".mp4", "bool", _("Allow .mp4"), True)] + + +At runtime the desired config values can be retrieved with ``self.getConfig(name)`` and setted with +``self.setConfig(name, value)``. Tagging Guidelines ------------------ +To categorize a plugin, a list of keywords can be assigned via ``__tags__`` attribute. You may add arbitrary +tags as you like, but please look at this table first to choose your tags. With standardised keywords we can generate +a better overview of the plugins and provide some search criteria. + +=============== =========================================================== +Keyword Meaning +=============== =========================================================== +image Anything related to image(hoster) +video Anything related to video(hoster) +captcha A plugin that needs captcha decrypting +interaction A plugin that makes uses of interaction with user +free A hoster without any premium service +premium_only A hoster only useable with account +ip_check A hoster that checks ip, that can be avoided with reconnect +=============== =========================================================== Basic Methods ------------- +All methods can be looked up at :class:`Base `. To note some important ones: + +The pyload core instance is accessible at ``self.core`` attribute +and the :class:`Api ` at ``self.core.api`` + +With ``self.load(...)`` you can load any url and get the result. This method is only available to Hoster and Crypter. +For other plugins use ``getURL(...)`` or ``getRequest()``. + +Use ``self.store(...)`` and ``self.retrieve(...)`` to store data persistantly into the database. + +Make use of ``logInfo, logError, logWarning, logDebug`` for logging purposes. + Debugging --------- +One of the most important aspects in software programming is debugging. It is especially important +for plugins which heavily rely on external input, which is true for all hoster and crypter plugins. +To enable debugging functionality start pyLoad with ``-d`` option or enable it in the config. + +You should use ``self.logDebug(msg)`` when ever it is reasonable. It is a good pratice to log server output +or the calculation of results and then check in the log if it really it what you are expecting. + +For further debugging you can install ipython [2]_, and set breakpoints with ``self.core.breakpoint()``. +It will open the python debugger [3]_ and pause the plugin thread. +To open a ipython shell in the running programm use ``self.shell()``. +These methods are usefull to gain access to the code flow at runtime and check or modify variables. + + +.. rubric:: Footnotes +.. [1] http://docs.python.org/library/gettext.html +.. [2] http://ipython.org/ +.. [3] http://docs.python.org/library/pdb.html \ No newline at end of file diff --git a/docs/plugins/crypter_plugin.rst b/docs/plugins/crypter_plugin.rst index d910ec412..639d58abf 100644 --- a/docs/plugins/crypter_plugin.rst +++ b/docs/plugins/crypter_plugin.rst @@ -3,3 +3,23 @@ Crypter - Extract links from pages ================================== +What about Decrypter and Container plugins? +Well, they work nearly the same, only that the function they have to provide is named ``decrypt`` + +Example: :: + + from module.plugin.Crypter import Crypter + + class MyFileCrypter(Crypter): + """ + plugin code + """ + def decrypt(self, pyfile): + + urls = ["http://get.pyload.org/src", "http://get.pyload.org/debian", "http://get.pyload.org/win"] + + self.packages.append(("pyLoad packages", urls, "pyLoad packages")) # urls list of urls + +They can access all the methods from :class:`Plugin `, but the important thing is they +have to append all packages they parsed to the `self.packages` list. Simply append tuples with `(name, urls, folder)`, +where urls is the list of urls contained in the packages. Thats all of your work, pyLoad will know what to do with them. diff --git a/docs/plugins/hoster_plugin.rst b/docs/plugins/hoster_plugin.rst index 59f35f5cb..e4575a001 100644 --- a/docs/plugins/hoster_plugin.rst +++ b/docs/plugins/hoster_plugin.rst @@ -3,39 +3,6 @@ Hoster - Load files to disk =========================== -A Plugin is a python file located at one of the subfolders in :file:`module/plugins/`. Either :file:`hoster`, :file:`crypter` -or :file:`container`, depending of it's type. - -There are three kinds of different plugins: **Hoster**, **Crypter**, **Container**. -All kind of plugins inherit from the base :class:`Plugin `. You should know its -convenient methods, they make your work easier ;-) - -Every plugin defines a ``__pattern__`` and when the user adds urls, every url is matched against the pattern defined in -the plugin. In case the ``__pattern__`` matched on the url the plugin will be assigned to handle it and instanciated when -pyLoad begins to download/decrypt the url. - -Plugin header -------------- - -How basic hoster plugin header could look like: :: - - from module.plugin.Hoster import Hoster - - class MyFileHoster(Hoster): - __version__ = "0.1" - __pattern__ = r"http://myfilehoster.example.com/file_id/[0-9]+" - __config__ = [] - -You have to define these meta-data, ``__pattern__`` has to be a regexp that sucessfully compiles with -``re.compile(__pattern__)``. - -Just like :ref:`write_hooks` you can add and use config values exatly the same way. -If you want a Crypter or Container plugin, just replace the word Hoster with your desired plugin type. - - -Hoster plugins --------------- - We head to the next important section, the ``process`` method of your plugin. In fact the ``process`` method is the only functionality your plugin has to provide, but its always a good idea to split up tasks to not produce spaghetti code. An example ``process`` function could look like this :: @@ -61,40 +28,23 @@ You need to know about the :class:`PyFile ` class, since a Some tasks your plugin should handle: proof if file is online, get filename, wait if needed, download the file, etc.. Wait times -__________ +---------- Some hoster require you to wait a specific time. Just set the time with ``self.setWait(seconds)`` or ``self.setWait(seconds, True)`` if you want pyLoad to perform a reconnect if needed. Captcha decrypting -__________________ +------------------ To handle captcha input just use ``self.decryptCaptcha(url, ...)``, it will be send to clients or handled by :class:`Hook ` plugins -Crypter -------- - -What about Decrypter and Container plugins? -Well, they work nearly the same, only that the function they have to provide is named ``decrypt`` +User interaction +---------------- -Example: :: - - from module.plugin.Crypter import Crypter - - class MyFileCrypter(Crypter): - """ - plugin code - """ - def decrypt(self, pyfile): - - urls = ["http://get.pyload.org/src", "http://get.pyload.org/debian", "http://get.pyload.org/win"] - - self.packages.append(("pyLoad packages", urls, "pyLoad packages")) # urls list of urls +Testing +------- -They can access all the methods from :class:`Plugin `, but the important thing is they -have to append all packages they parsed to the `self.packages` list. Simply append tuples with `(name, urls, folder)`, -where urls is the list of urls contained in the packages. Thats all of your work, pyLoad will know what to do with them. Examples -------- diff --git a/docs/plugins/overview.rst b/docs/plugins/overview.rst index 23913b787..68ad96dfd 100755 --- a/docs/plugins/overview.rst +++ b/docs/plugins/overview.rst @@ -14,7 +14,7 @@ Extending pyLoad pyLoad offers an comfortable and powerful plugin system to make extending possible. With it you only need to have some python knowledge and can just start right away writing your own plugins. This document gives you an overwiew about the -conceptual part. You should not left out the `Base` part, since it contains basic functionality for all plugins types. +conceptual part. You should not left out the :doc:`Base ` part, since it contains basic functionality for all plugins types. .. rubric:: Contents -- cgit v1.2.3 From 7df4718276a12b7f19a73d3b789c791d57bf4342 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 29 Jan 2012 22:57:41 +0100 Subject: doc for crypter plugin --- docs/plugins/base_plugin.rst | 19 ++++++++++----- docs/plugins/crypter_plugin.rst | 54 ++++++++++++++++++++++++++++++++--------- 2 files changed, 55 insertions(+), 18 deletions(-) (limited to 'docs') diff --git a/docs/plugins/base_plugin.rst b/docs/plugins/base_plugin.rst index e3c72994b..1849f3986 100644 --- a/docs/plugins/base_plugin.rst +++ b/docs/plugins/base_plugin.rst @@ -17,11 +17,15 @@ An overview of acceptible values can be found in :class:`Base ` and :class:`Crypter ` +needs to have a specific regexp [1]_ ``__pattern__``. This will be matched against input urls and if a suited +plugin is found it is selected to handle the url. + You don't need to subclass :class:`Base ` directly, but the intermediate type according to your plugin. As example we choose an Hoster plugin, but the same is true for all plugin types. -For localization pyLoad supports gettext [1]_, to mark strings for translation surround them with ``_("...")``. +For localization pyLoad supports gettext [2]_, to mark strings for translation surround them with ``_("...")``. How basic hoster plugin header could look like:: @@ -34,6 +38,8 @@ How basic hoster plugin header could look like:: is not needed for hoster plugin, but hook plugin should have it so the user knows what they doing.""") +In future examples the meta data will be left out, but remember it's required in every plugin! + Config Entries -------------- @@ -96,13 +102,14 @@ To enable debugging functionality start pyLoad with ``-d`` option or enable it i You should use ``self.logDebug(msg)`` when ever it is reasonable. It is a good pratice to log server output or the calculation of results and then check in the log if it really it what you are expecting. -For further debugging you can install ipython [2]_, and set breakpoints with ``self.core.breakpoint()``. -It will open the python debugger [3]_ and pause the plugin thread. +For further debugging you can install ipython [3]_, and set breakpoints with ``self.core.breakpoint()``. +It will open the python debugger [4]_ and pause the plugin thread. To open a ipython shell in the running programm use ``self.shell()``. These methods are usefull to gain access to the code flow at runtime and check or modify variables. .. rubric:: Footnotes -.. [1] http://docs.python.org/library/gettext.html -.. [2] http://ipython.org/ -.. [3] http://docs.python.org/library/pdb.html \ No newline at end of file +.. [1] http://docs.python.org/library/re.html +.. [2] http://docs.python.org/library/gettext.html +.. [3] http://ipython.org/ +.. [4] http://docs.python.org/library/pdb.html \ No newline at end of file diff --git a/docs/plugins/crypter_plugin.rst b/docs/plugins/crypter_plugin.rst index 639d58abf..1497ced07 100644 --- a/docs/plugins/crypter_plugin.rst +++ b/docs/plugins/crypter_plugin.rst @@ -3,23 +3,53 @@ Crypter - Extract links from pages ================================== -What about Decrypter and Container plugins? -Well, they work nearly the same, only that the function they have to provide is named ``decrypt`` - -Example: :: +We are starting with the simplest plugin, the :class:`Crypter `. +It's job is it to take a url as input and generate new package or links, for example by filtering the urls or +loading a page and extracting links from the html code. You need to define the ``__pattern__`` to match +target urls and subclass from :class:`Crypter `. :: from module.plugin.Crypter import Crypter class MyFileCrypter(Crypter): - """ - plugin code - """ - def decrypt(self, pyfile): + __pattern__ = r"mycrypter.com/id/([0-9]+)" + + def decryptURL(self, url): urls = ["http://get.pyload.org/src", "http://get.pyload.org/debian", "http://get.pyload.org/win"] + return urls + +You have to overwrite at least one of ``.decryptFile``, ``.decryptURL``, ``.decryptURLs``. The first one +is only useful for container files, whereas the last is usefull when it's possible to handle a bunch of urls +at once. If in doubt, just overwrite `decryptURL`. + +Generating Packages +------------------- + +When finished with decrypting just return the urls as list and they will be added to the package. You can also +create new Packages if needed by instantiating a :class:`Package` instance, which will look like the following:: + + from module.plugin.Crypter import Crypter, Package + + class MyFileCrypter(Crypter): + + def decryptURL(self, url): + + html = self.load(url) + + # .decrypt_from_content is only a example method here and will return a list of urls + urls = self.decrypt_from_content(html) + return Package("my new package", urls) + +And that's basically all you need to know. Just as little side-note if you want to use decrypter in +your code you can use:: + + plugin = self.core.pluginManager.loadClass("crypter", "NameOfThePlugin") + # decrypted will be a list of urls + decrypted = plugin.decrypt(urls) - self.packages.append(("pyLoad packages", urls, "pyLoad packages")) # urls list of urls +Testing +------- -They can access all the methods from :class:`Plugin `, but the important thing is they -have to append all packages they parsed to the `self.packages` list. Simply append tuples with `(name, urls, folder)`, -where urls is the list of urls contained in the packages. Thats all of your work, pyLoad will know what to do with them. +Please append a test link at :file:`tests/crypterlinks.txt` followed by `||xy`, where xy is the number of +expected links/packages to extract. +Our testrunner will be able to check your plugin periodical for functionality. \ No newline at end of file -- cgit v1.2.3 From da4cf026ad116518fefc3429b74a8cd94aeef73f Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 5 Feb 2012 17:27:13 +0100 Subject: updated documentation + diagrams --- docs/index.rst | 2 ++ docs/module_overview.rst | 2 ++ docs/plugins/base_plugin.rst | 24 +++++++++++++----------- docs/plugins/crypter_plugin.rst | 18 ++++++++++++++++-- docs/plugins/hoster_plugin.rst | 5 +++++ docs/plugins/overview.rst | 5 ++++- docs/system/hoster_diagrams.rst | 16 ++++++++++++++++ docs/system/overview.rst | 25 +++++++++++++++++++++++++ docs/system/plugin_hierarchy.rst | 9 +++++++++ docs/system/pyload_PluginHierarchy.png | Bin 0 -> 23553 bytes docs/system/pyload_ad_Hoster.png | Bin 0 -> 34237 bytes docs/system/pyload_sd_Hoster.png | Bin 0 -> 28501 bytes 12 files changed, 92 insertions(+), 14 deletions(-) create mode 100644 docs/system/hoster_diagrams.rst create mode 100755 docs/system/overview.rst create mode 100644 docs/system/plugin_hierarchy.rst create mode 100644 docs/system/pyload_PluginHierarchy.png create mode 100644 docs/system/pyload_ad_Hoster.png create mode 100644 docs/system/pyload_sd_Hoster.png (limited to 'docs') diff --git a/docs/index.rst b/docs/index.rst index 31d688e65..b18f068f2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -31,6 +31,8 @@ Contents api/overview.rst plugins/overview.rst + system/overview.rst + module_overview.rst .. currentmodule:: module diff --git a/docs/module_overview.rst b/docs/module_overview.rst index b8db51538..1e771f36a 100644 --- a/docs/module_overview.rst +++ b/docs/module_overview.rst @@ -10,7 +10,9 @@ A little selection of most important modules in pyLoad. module.Api.Api module.plugins.Base.Base module.plugins.Hoster.Hoster + module.plugins.internal.SimpleHoster.SimpleHoster module.plugins.Crypter.Crypter + module.plugins.internal.SimpleCrypter.SimpleCrypter module.plugins.Hook.Hook module.plugins.Account.Account module.plugins.MultiHoster.MultiHoster diff --git a/docs/plugins/base_plugin.rst b/docs/plugins/base_plugin.rst index 1849f3986..911f5d429 100644 --- a/docs/plugins/base_plugin.rst +++ b/docs/plugins/base_plugin.rst @@ -5,8 +5,9 @@ Base Plugin - And here it begins... A Plugin in pyLoad is a python file located at one of the subfolders in :file:`module/plugins/`. All different plugin types inherit from :class:`Base `, which defines basic methods -and meta data. You should read this section carefully, because it's the base for all plugin development. -After that it is a good idea to look at several already existing plugin to get a more detailed idea of how +and meta data. You should read this section carefully, because it's the base for all plugin development. It +is also a good idea to look at the class diagram [1]_ for all plugin types to get an overview. +At last you should look at several already existing plugin to get a more detailed idea of how they have to look like and whats possible with them. Meta Data @@ -18,15 +19,15 @@ Non needed attributes can be left out, except ``__version__``. Nevertheless plea as you can, when you want to submit your plugin to the public repo. Additionally :class:`Crypter ` and :class:`Crypter ` -needs to have a specific regexp [1]_ ``__pattern__``. This will be matched against input urls and if a suited +needs to have a specific regexp [2]_ ``__pattern__``. This will be matched against input urls and if a suited plugin is found it is selected to handle the url. +For localization pyLoad supports gettext [3]_, to mark strings for translation surround them with ``_("...")``. + You don't need to subclass :class:`Base ` directly, but the intermediate type according to your plugin. As example we choose an Hoster plugin, but the same is true for all plugin types. -For localization pyLoad supports gettext [2]_, to mark strings for translation surround them with ``_("...")``. - How basic hoster plugin header could look like:: from module.plugin.Hoster import Hoster @@ -102,14 +103,15 @@ To enable debugging functionality start pyLoad with ``-d`` option or enable it i You should use ``self.logDebug(msg)`` when ever it is reasonable. It is a good pratice to log server output or the calculation of results and then check in the log if it really it what you are expecting. -For further debugging you can install ipython [3]_, and set breakpoints with ``self.core.breakpoint()``. -It will open the python debugger [4]_ and pause the plugin thread. +For further debugging you can install ipython [4]_, and set breakpoints with ``self.core.breakpoint()``. +It will open the python debugger [5]_ and pause the plugin thread. To open a ipython shell in the running programm use ``self.shell()``. These methods are usefull to gain access to the code flow at runtime and check or modify variables. .. rubric:: Footnotes -.. [1] http://docs.python.org/library/re.html -.. [2] http://docs.python.org/library/gettext.html -.. [3] http://ipython.org/ -.. [4] http://docs.python.org/library/pdb.html \ No newline at end of file +.. [1] :ref:`plugin_hierarchy` +.. [2] http://docs.python.org/library/re.html +.. [3] http://docs.python.org/library/gettext.html +.. [4] http://ipython.org/ +.. [5] http://docs.python.org/library/pdb.html \ No newline at end of file diff --git a/docs/plugins/crypter_plugin.rst b/docs/plugins/crypter_plugin.rst index 1497ced07..4e7803808 100644 --- a/docs/plugins/crypter_plugin.rst +++ b/docs/plugins/crypter_plugin.rst @@ -19,7 +19,7 @@ target urls and subclass from :class:`Crypter `. return urls You have to overwrite at least one of ``.decryptFile``, ``.decryptURL``, ``.decryptURLs``. The first one -is only useful for container files, whereas the last is usefull when it's possible to handle a bunch of urls +is only useful for container files, whereas the last is useful when it's possible to handle a bunch of urls at once. If in doubt, just overwrite `decryptURL`. Generating Packages @@ -44,8 +44,22 @@ And that's basically all you need to know. Just as little side-note if you want your code you can use:: plugin = self.core.pluginManager.loadClass("crypter", "NameOfThePlugin") + # Core instance is needed for decrypting # decrypted will be a list of urls - decrypted = plugin.decrypt(urls) + decrypted = plugin.decrypt(core, urls) + + +SimpleCrypter +------------- + +For simple crypter services there is the :class:`SimpleCrypter ` class which handles most of the workflow. Only the regexp +pattern have to be defined. + +Exmaple:: + + from module.plugins.internal.SimpleCrypter import SimpleCrypter + + class MyFileCrypter(SimpleCrypter): Testing ------- diff --git a/docs/plugins/hoster_plugin.rst b/docs/plugins/hoster_plugin.rst index e4575a001..ee112b570 100644 --- a/docs/plugins/hoster_plugin.rst +++ b/docs/plugins/hoster_plugin.rst @@ -42,6 +42,11 @@ or handled by :class:`Hook ` plugins User interaction ---------------- + +SimpleHoster +------------ + + Testing ------- diff --git a/docs/plugins/overview.rst b/docs/plugins/overview.rst index 68ad96dfd..0388db7e2 100755 --- a/docs/plugins/overview.rst +++ b/docs/plugins/overview.rst @@ -15,6 +15,7 @@ Extending pyLoad pyLoad offers an comfortable and powerful plugin system to make extending possible. With it you only need to have some python knowledge and can just start right away writing your own plugins. This document gives you an overwiew about the conceptual part. You should not left out the :doc:`Base ` part, since it contains basic functionality for all plugins types. +A class diagram visualizing the relationship can be find below [1]_ .. rubric:: Contents @@ -28,4 +29,6 @@ conceptual part. You should not left out the :doc:`Base ` part, sin -.. rubric:: Footnotes \ No newline at end of file +.. rubric:: Footnotes + +.. [1] :ref:`plugin_hierarchy` \ No newline at end of file diff --git a/docs/system/hoster_diagrams.rst b/docs/system/hoster_diagrams.rst new file mode 100644 index 000000000..313f75c57 --- /dev/null +++ b/docs/system/hoster_diagrams.rst @@ -0,0 +1,16 @@ +.. _hoster_diagrams: + +=============== +Hoster Workflow +=============== + +The basic workflow of a hoster plugin. This is only a generalization, in most cases it is more complex +and order will differ. + +Activity Diagram +---------------- +.. image:: pyload_ad_Hoster.png + +Sequence Diagram +---------------- +.. image:: pyload_sd_Hoster.png \ No newline at end of file diff --git a/docs/system/overview.rst b/docs/system/overview.rst new file mode 100755 index 000000000..00e439f45 --- /dev/null +++ b/docs/system/overview.rst @@ -0,0 +1,25 @@ +.. _overview: + +============= +System Design +============= + +.. pull-quote:: + Programs must be written for people to read, and only incidentally for machines to execute. + + -- H. Abelson and G. Sussman + + +.. rubric:: Motivation + +In this section you will find information and diagrams to better understand the concept of pyload. + +.. rubric:: Contents + +.. toctree:: + + plugin_hierarchy.rst + hoster_diagrams.rst + + +.. rubric:: Footnotes \ No newline at end of file diff --git a/docs/system/plugin_hierarchy.rst b/docs/system/plugin_hierarchy.rst new file mode 100644 index 000000000..b663943a3 --- /dev/null +++ b/docs/system/plugin_hierarchy.rst @@ -0,0 +1,9 @@ +.. _plugin_hierarchy: + +================ +Plugin Hierarchy +================ + +Class diagram that describes plugin relationships. + +.. image:: pyload_PluginHierarchy.png \ No newline at end of file diff --git a/docs/system/pyload_PluginHierarchy.png b/docs/system/pyload_PluginHierarchy.png new file mode 100644 index 000000000..f1a753ee2 Binary files /dev/null and b/docs/system/pyload_PluginHierarchy.png differ diff --git a/docs/system/pyload_ad_Hoster.png b/docs/system/pyload_ad_Hoster.png new file mode 100644 index 000000000..0ee064edc Binary files /dev/null and b/docs/system/pyload_ad_Hoster.png differ diff --git a/docs/system/pyload_sd_Hoster.png b/docs/system/pyload_sd_Hoster.png new file mode 100644 index 000000000..e629a1949 Binary files /dev/null and b/docs/system/pyload_sd_Hoster.png differ -- cgit v1.2.3 From 4df2b77fdf42046fe19bd371be7c7255986b5980 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 6 Mar 2012 13:36:39 +0100 Subject: renamed hooks to addons, new filemanager and database, many new api methods you will loose ALL your LINKS, webinterface will NOT work --- docs/plugins/addon_plugin.rst | 162 ++++++++++++++++++++++++++++++++++++++++++ docs/plugins/hook_plugin.rst | 162 ------------------------------------------ docs/plugins/overview.rst | 2 +- 3 files changed, 163 insertions(+), 163 deletions(-) create mode 100644 docs/plugins/addon_plugin.rst delete mode 100644 docs/plugins/hook_plugin.rst (limited to 'docs') diff --git a/docs/plugins/addon_plugin.rst b/docs/plugins/addon_plugin.rst new file mode 100644 index 000000000..57c7e4a96 --- /dev/null +++ b/docs/plugins/addon_plugin.rst @@ -0,0 +1,162 @@ +.. _write_addons: + +Addon - Add new functionality +============================= + +A Hook is a python file which is located at :file:`module/plugins/hooks`. +The :class:`HookManager ` will load it automatically on startup. Only one instance exists +over the complete lifetime of pyload. Your hook can interact on various events called by the :class:`HookManager `, +do something complete autonomic and has full access to the :class:`Api ` and every detail of pyLoad. +The UpdateManager, CaptchaTrader, UnRar and many more are realised as hooks. + +Hook header +----------- + +Your hook needs to subclass :class:`Hook ` and will inherit all of its method, make sure to check its documentation! + +All Hooks should start with something like this: :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + __name__ = "YourHook" + __version__ = "0.1" + __description__ = "Does really cool stuff" + __config__ = [ ("activated" , "bool" , "Activated" , "True" ) ] + __threaded__ = ["downloadFinished"] + __author_name__ = ("Me") + __author_mail__ = ("me@has-no-mail.com") + +All meta-data is defined in the header, you need at least one option at ``__config__`` so the user can toggle your +hook on and off. Dont't overwrite the ``init`` method if not neccesary, use ``setup`` instead. + +Using the Config +---------------- + +We are taking a closer look at the ``__config__`` parameter. +You can add more config values as desired by adding tuples of the following format to the config list: ``("name", "type", "description", "default value")``. +When everything went right you can access the config values with ``self.getConfig(name)`` and ``self.setConfig(name,value``. + + +Interacting on Events +--------------------- + +The next step is to think about where your Hook action takes places. + +The easiest way is to overwrite specific methods defined by the :class:`Hook ` base class. +The name is indicating when the function gets called. +See :class:`Hook ` page for a complete listing. + +You should be aware of the arguments the Hooks are called with, whether its a :class:`PyFile ` +or :class:`PyPackage ` you should read its related documentation to know how to access her great power and manipulate them. + +A basic excerpt would look like: :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + + def coreReady(self): + print "Yay, the core is ready let's do some work." + + def downloadFinished(self, pyfile): + print "A Download just finished." + +Another important feature to mention can be seen at the ``__threaded__`` parameter. Function names listed will be executed +in a thread, in order to not block the main thread. This should be used for all kind of longer processing tasks. + +Another and more flexible and powerful way is to use event listener. +All hook methods exists as event and very useful additional events are dispatched by the core. For a little overview look +at :class:`HookManager `. Keep in mind that you can define own events and other people may listen on them. + +For your convenience it's possible to register listeners automatical via the ``event_map`` attribute. +It requires a `dict` that maps event names to function names or a `list` of function names. It's important that all names are strings :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + event_map = {"downloadFinished" : "doSomeWork", + "allDownloadsFnished": "someMethod", + "coreReady": "initialize"} + + def initialize(self): + print "Initialized." + + def doSomeWork(self, pyfile): + print "This is equivalent to the above example." + + def someMethod(self): + print "The underlying event (allDownloadsFinished) for this method is not available through the base class" + +An advantage of the event listener is that you are able to register and remove the listeners at runtime. +Use `self.manager.addEvent("name", function)`, `self.manager.removeEvent("name", function)` and see doc for +:class:`HookManager `. Contrary to ``event_map``, ``function`` has to be a reference +and **not** a `string`. + +We introduced events because it scales better if there a a huge amount of events and hooks. So all future interaction will be exclusive +available as event and not accessible through overwriting hook methods. However you can safely do this, it will not be removed and is easier to implement. + + +Providing RPC services +---------------------- + +You may noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. +So probably clients want to be able to interact with your hook to request it's state or invoke some action. + +Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: + + from module.plugins.Hook import Hook, Expose + + class YourHook(Hook): + """ + Your Hook code here. + """ + + @Expose + def invoke(self, arg): + print "Invoked with", arg + +Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. +Here is a basic example: :: + + #Assuming client is a ThriftClient or Api object + + print client.getServices() + print client.call(ServiceCall("YourHook", "invoke", "an argument")) + +Providing status information +---------------------------- +Your hook can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. + +Just store everything in ``self.info``. :: + + from module.plugins.Hook import Hook + + class YourHook(Hook): + """ + Your Hook code here. + """ + + def setup(self): + self.info = {"running": False} + + def coreReady(self): + self.info["running"] = True + +Usable with: :: + + #Assuming client is a ThriftClient or Api object + + print client.getAllInfo() + +Example +------- + Sorry but you won't find an example here ;-) + + Look at :file:`module/plugins/hooks` and you will find plenty examples there. diff --git a/docs/plugins/hook_plugin.rst b/docs/plugins/hook_plugin.rst deleted file mode 100644 index be1097057..000000000 --- a/docs/plugins/hook_plugin.rst +++ /dev/null @@ -1,162 +0,0 @@ -.. _write_hooks: - -Hook - Do whatever you want -============================= - -A Hook is a python file which is located at :file:`module/plugins/hooks`. -The :class:`HookManager ` will load it automatically on startup. Only one instance exists -over the complete lifetime of pyload. Your hook can interact on various events called by the :class:`HookManager `, -do something complete autonomic and has full access to the :class:`Api ` and every detail of pyLoad. -The UpdateManager, CaptchaTrader, UnRar and many more are realised as hooks. - -Hook header ------------ - -Your hook needs to subclass :class:`Hook ` and will inherit all of its method, make sure to check its documentation! - -All Hooks should start with something like this: :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - __name__ = "YourHook" - __version__ = "0.1" - __description__ = "Does really cool stuff" - __config__ = [ ("activated" , "bool" , "Activated" , "True" ) ] - __threaded__ = ["downloadFinished"] - __author_name__ = ("Me") - __author_mail__ = ("me@has-no-mail.com") - -All meta-data is defined in the header, you need at least one option at ``__config__`` so the user can toggle your -hook on and off. Dont't overwrite the ``init`` method if not neccesary, use ``setup`` instead. - -Using the Config ----------------- - -We are taking a closer look at the ``__config__`` parameter. -You can add more config values as desired by adding tuples of the following format to the config list: ``("name", "type", "description", "default value")``. -When everything went right you can access the config values with ``self.getConfig(name)`` and ``self.setConfig(name,value``. - - -Interacting on Events ---------------------- - -The next step is to think about where your Hook action takes places. - -The easiest way is to overwrite specific methods defined by the :class:`Hook ` base class. -The name is indicating when the function gets called. -See :class:`Hook ` page for a complete listing. - -You should be aware of the arguments the Hooks are called with, whether its a :class:`PyFile ` -or :class:`PyPackage ` you should read its related documentation to know how to access her great power and manipulate them. - -A basic excerpt would look like: :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - """ - Your Hook code here. - """ - - def coreReady(self): - print "Yay, the core is ready let's do some work." - - def downloadFinished(self, pyfile): - print "A Download just finished." - -Another important feature to mention can be seen at the ``__threaded__`` parameter. Function names listed will be executed -in a thread, in order to not block the main thread. This should be used for all kind of longer processing tasks. - -Another and more flexible and powerful way is to use event listener. -All hook methods exists as event and very useful additional events are dispatched by the core. For a little overview look -at :class:`HookManager `. Keep in mind that you can define own events and other people may listen on them. - -For your convenience it's possible to register listeners automatical via the ``event_map`` attribute. -It requires a `dict` that maps event names to function names or a `list` of function names. It's important that all names are strings :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - """ - Your Hook code here. - """ - event_map = {"downloadFinished" : "doSomeWork", - "allDownloadsFnished": "someMethod", - "coreReady": "initialize"} - - def initialize(self): - print "Initialized." - - def doSomeWork(self, pyfile): - print "This is equivalent to the above example." - - def someMethod(self): - print "The underlying event (allDownloadsFinished) for this method is not available through the base class" - -An advantage of the event listener is that you are able to register and remove the listeners at runtime. -Use `self.manager.addEvent("name", function)`, `self.manager.removeEvent("name", function)` and see doc for -:class:`HookManager `. Contrary to ``event_map``, ``function`` has to be a reference -and **not** a `string`. - -We introduced events because it scales better if there a a huge amount of events and hooks. So all future interaction will be exclusive -available as event and not accessible through overwriting hook methods. However you can safely do this, it will not be removed and is easier to implement. - - -Providing RPC services ----------------------- - -You may noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. -So probably clients want to be able to interact with your hook to request it's state or invoke some action. - -Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: - - from module.plugins.Hook import Hook, Expose - - class YourHook(Hook): - """ - Your Hook code here. - """ - - @Expose - def invoke(self, arg): - print "Invoked with", arg - -Thats all, it's available via the :class:`Api ` now. If you want to use it read :ref:`access_api`. -Here is a basic example: :: - - #Assuming client is a ThriftClient or Api object - - print client.getServices() - print client.call(ServiceCall("YourHook", "invoke", "an argument")) - -Providing status information ----------------------------- -Your hook can store information in a ``dict`` that can easily be retrievied via the :class:`Api `. - -Just store everything in ``self.info``. :: - - from module.plugins.Hook import Hook - - class YourHook(Hook): - """ - Your Hook code here. - """ - - def setup(self): - self.info = {"running": False} - - def coreReady(self): - self.info["running"] = True - -Usable with: :: - - #Assuming client is a ThriftClient or Api object - - print client.getAllInfo() - -Example -------- - Sorry but you won't find an example here ;-) - - Look at :file:`module/plugins/hooks` and you will find plenty examples there. diff --git a/docs/plugins/overview.rst b/docs/plugins/overview.rst index 0388db7e2..70db5ac90 100755 --- a/docs/plugins/overview.rst +++ b/docs/plugins/overview.rst @@ -25,7 +25,7 @@ A class diagram visualizing the relationship can be find below [1]_ crypter_plugin.rst hoster_plugin.rst account_plugin.rst - hook_plugin.rst + addon_plugin.rst -- cgit v1.2.3 From 5a7f415a25d8e036a37851fcd5e9be81caae2804 Mon Sep 17 00:00:00 2001 From: X3n0m0rph59 Date: Sun, 22 Apr 2012 16:54:06 +0200 Subject: Fixed spelling in the documentation --- docs/api/json_api.rst | 36 ++++++++++++++++++------------------ docs/api/overview.rst | 2 +- docs/api/thrift_api.rst | 6 +++--- docs/plugins/addon_plugin.rst | 33 +++++++++++++++++---------------- docs/plugins/base_plugin.rst | 36 ++++++++++++++++++------------------ docs/plugins/crypter_plugin.rst | 8 ++++---- docs/plugins/hoster_plugin.rst | 8 ++++---- docs/plugins/overview.rst | 8 ++++---- 8 files changed, 69 insertions(+), 68 deletions(-) (limited to 'docs') diff --git a/docs/api/json_api.rst b/docs/api/json_api.rst index 3df006c49..6d55b13b4 100644 --- a/docs/api/json_api.rst +++ b/docs/api/json_api.rst @@ -4,35 +4,35 @@ JSON API ======== -JSON [1]_ is a lightweight object notation and wrapper exists for nearly every programming language. Every +JSON [1]_ is a lightweight object notation and wrappers exist for nearly every programming language. Every modern browser is able to load JSON objects with JavaScript. Unlike to thrift you don't need to generate or precompile -any stub methods, the JSON :class:`Api ` is ready to use for most language. The libary is really lightweight (at least in python) -and you can build very lightweight scripts with it. Because of the builtin support JSON is the first choice for all browser +any stub methods, the JSON :class:`Api ` is ready to be used in most languages. The library is really lightweight (at least in python) +and you can build very lightweight scripts with it. Because of the builtin support, JSON is the first choice for all browser applications. In our case JSON is just the output format, you have exactly the same methods available as with the thrift backend. The only -difference is the used protocol. +difference is the underlying protocol. -So are there still reasons to choose the original :doc:`thrift ` backend in favor to JSON? Yes, since it +Are there still reasons to choose the original :doc:`thrift ` backend in favor to JSON? Yes, since it uses a binary protocol the performance will be better (when generating the objects), traffic will be smaller and therefore the transfer faster. In most IDEs you will get code completion, because of the pre-generated classes, which can make work much easier. If you intend to write a full client you should prefer thrift if the language is supported, for lightweight scripts and -in browser environment JSON wil be the better choice. +in browser environments JSON will be the better choice. Login ----- -First you need to authenticate, if you using this within the webinterface and the user is logged the API is also accessible, +First you need to authenticate, if you are using this within the web interface and the user is logged in, the API is also accessible, since they share the same cookie/session. -However, if you are building a external client and want to authenticate manually +However, if you are building an external client and want to authenticate manually you have to send your credentials ``username`` and ``password`` as POST parameter to ``http://pyload-core/api/login``. The result will be your session id. If you are using cookies, it will be set and you can use the API now. -In case you dont't have cookies enabled you can pass the session id as ``session`` POST parameter +In case you don't have cookies enabled you can pass the session id as ``session`` POST parameter so pyLoad can authenticate you. @@ -40,19 +40,19 @@ Calling Methods --------------- In general you can use any method listed at the :class:`Api ` documentation, which is also available to -the thriftbackend. +the thrift backend. Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address -or hostname including the webinterface port. By default on local access this would be `localhost:8000`. +or hostname including the web interface port. By default on local access this would be `localhost:8000`. -The return value will be formatted in JSON, complex data types as dictionaries. Definition for datatypes can be found +The return value will be formatted in JSON, complex data types as dictionaries. Definition for data types can be found :doc:`here ` Passing parameters ------------------ -To pass arguments you have two choices. -Either use positional arguments, eg ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword +To pass arguments you have two choices: +Either use positional arguments, e.g.: ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword arguments supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names in the :class:`Api ` documentation. @@ -60,11 +60,11 @@ It is important that *all* arguments are in JSON format. So ``http://pyload-core 1 represents an integer in json format. On the other hand if the method is expecting strings, this would be correct: ``http://pyload-core/api/getUserData/"username"/"password"``. -Strings are wrapped in double qoutes, because `"username"` represents a string in json format. It's not limited to -strings and intergers, every container type like lists and dicts are possible. You usually don't have to convert them. -Just use a json encoder before using them in the HTTP request. +Strings are wrapped in double qoutes, because `"username"` represents a string in JSON format. It's not limited to +strings and integers, every container type like lists and dicts are possible. You usually don't have to convert them. +Just use a JSON encoder before using them in the HTTP request. -Please note that the data have to be urlencoded at last. (Most libaries will do that automatically) +Please note that the data has to be urlencoded at last. (Most libraries will do that automatically) .. rubric:: Footnotes diff --git a/docs/api/overview.rst b/docs/api/overview.rst index 47fe1be82..37aa8a526 100644 --- a/docs/api/overview.rst +++ b/docs/api/overview.rst @@ -16,7 +16,7 @@ The idea of the centralized pyLoad :class:`Api ` is to give unif and plugins in pyLoad, and furthermore to other clients, written in arbitrary programming languages. Most of the :class:`Api ` functionality is exposed via RPC [2]_ and accessible via thrift [3]_ or simple JSON objects [4]_. In conclusion the :class:`Api ` is accessible via many programming language, -over network from remote maschines and over browser with javascript. +over network from remote machines and over browser with javascript. .. rubric:: Contents diff --git a/docs/api/thrift_api.rst b/docs/api/thrift_api.rst index a4987a797..cd1d2f23c 100644 --- a/docs/api/thrift_api.rst +++ b/docs/api/thrift_api.rst @@ -5,8 +5,8 @@ Thrift API ========== Thrift [1]_ was first developed in-house at facebook, but later published to public domain and developed at Apache Incubator. -It includes a binary protocol for remote calls, which is much more performant than other data formats like XML, additionally -it is available for numerous languages and therefore we choosed it as primary backend for our API. +It includes a binary protocol for remote calls, which has a much better performance than other data formats like XML, additionally +it is available for numerous languages and therefore we choose it as primary backend for our API. First of all, you need to know what you can do with our API. It lets you do all common task like retrieving download status, manage queue, manage accounts, modify config and so on. @@ -15,7 +15,7 @@ This document is not intended to explain every function in detail, for a complet see :class:`Api `. Of course its possible to access the ``core.api`` attribute in plugins and hooks, but much more -interesting is the possibillity to call function from different programs written in many different languages. +interesting is the possibility to call function from different programs written in many different languages. pyLoad uses thrift as backend and provides its :class:`Api ` as service. More information about thrift can be found in their wiki [2]_. diff --git a/docs/plugins/addon_plugin.rst b/docs/plugins/addon_plugin.rst index 57c7e4a96..c2258f2aa 100644 --- a/docs/plugins/addon_plugin.rst +++ b/docs/plugins/addon_plugin.rst @@ -6,15 +6,15 @@ Addon - Add new functionality A Hook is a python file which is located at :file:`module/plugins/hooks`. The :class:`HookManager ` will load it automatically on startup. Only one instance exists over the complete lifetime of pyload. Your hook can interact on various events called by the :class:`HookManager `, -do something complete autonomic and has full access to the :class:`Api ` and every detail of pyLoad. -The UpdateManager, CaptchaTrader, UnRar and many more are realised as hooks. +do something completely autonomic and has full access to the :class:`Api ` and every detail of pyLoad. +The UpdateManager, CaptchaTrader, UnRar and many more are implemented as hooks. Hook header ----------- -Your hook needs to subclass :class:`Hook ` and will inherit all of its method, make sure to check its documentation! +Your hook needs to subclass :class:`Hook ` and will inherit all of its methods, so make sure to check it's documentation! -All Hooks should start with something like this: :: +All hooks should start with something like this: :: from module.plugins.Hook import Hook @@ -28,7 +28,7 @@ All Hooks should start with something like this: :: __author_mail__ = ("me@has-no-mail.com") All meta-data is defined in the header, you need at least one option at ``__config__`` so the user can toggle your -hook on and off. Dont't overwrite the ``init`` method if not neccesary, use ``setup`` instead. +hook on and off. Don't overwrite the ``init`` method if not necessary, use ``setup`` instead. Using the Config ---------------- @@ -41,16 +41,16 @@ When everything went right you can access the config values with ``self.getConfi Interacting on Events --------------------- -The next step is to think about where your Hook action takes places. +The next step is to think about where your Hook action takes place. The easiest way is to overwrite specific methods defined by the :class:`Hook ` base class. The name is indicating when the function gets called. See :class:`Hook ` page for a complete listing. -You should be aware of the arguments the Hooks are called with, whether its a :class:`PyFile ` -or :class:`PyPackage ` you should read its related documentation to know how to access her great power and manipulate them. +You should be aware of the arguments the hooks are called with, whether its a :class:`PyFile ` +or :class:`PyPackage ` you should read the relevant documentation to know how to access it's great power and manipulate them. -A basic excerpt would look like: :: +What a basic excerpt would look like: :: from module.plugins.Hook import Hook @@ -66,13 +66,13 @@ A basic excerpt would look like: :: print "A Download just finished." Another important feature to mention can be seen at the ``__threaded__`` parameter. Function names listed will be executed -in a thread, in order to not block the main thread. This should be used for all kind of longer processing tasks. +in a thread, in order to not block the main thread. This should be used for all kinds of long lived processing tasks. -Another and more flexible and powerful way is to use event listener. +Another and more flexible and powerful way is to use the event listener. All hook methods exists as event and very useful additional events are dispatched by the core. For a little overview look -at :class:`HookManager `. Keep in mind that you can define own events and other people may listen on them. +at :class:`HookManager `. Keep in mind that you can define your own events and other people may listen on them. -For your convenience it's possible to register listeners automatical via the ``event_map`` attribute. +For your convenience it's possible to register listeners automatically via the ``event_map`` attribute. It requires a `dict` that maps event names to function names or a `list` of function names. It's important that all names are strings :: from module.plugins.Hook import Hook @@ -99,14 +99,15 @@ Use `self.manager.addEvent("name", function)`, `self.manager.removeEvent("name", :class:`HookManager `. Contrary to ``event_map``, ``function`` has to be a reference and **not** a `string`. -We introduced events because it scales better if there a a huge amount of events and hooks. So all future interaction will be exclusive +We introduced events because it scales better if there is a huge amount of events and hooks. So all future interactions will be exclusively available as event and not accessible through overwriting hook methods. However you can safely do this, it will not be removed and is easier to implement. -Providing RPC services +Providing + RPC services ---------------------- -You may noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. +You may have noticed that pyLoad has an :class:`Api `, which can be used internal or called by clients via RPC. So probably clients want to be able to interact with your hook to request it's state or invoke some action. Sounds complicated but is very easy to do. Just use the ``Expose`` decorator: :: diff --git a/docs/plugins/base_plugin.rst b/docs/plugins/base_plugin.rst index 911f5d429..91a6eef44 100644 --- a/docs/plugins/base_plugin.rst +++ b/docs/plugins/base_plugin.rst @@ -8,43 +8,43 @@ All different plugin types inherit from :class:`Base ` and meta data. You should read this section carefully, because it's the base for all plugin development. It is also a good idea to look at the class diagram [1]_ for all plugin types to get an overview. At last you should look at several already existing plugin to get a more detailed idea of how -they have to look like and whats possible with them. +they have to look like and what is possible with them. Meta Data --------- All important data which must be known by pyLoad is set using class attributes pre- and suffixed with ``__``. -An overview of acceptible values can be found in :class:`Base ` source code. -Non needed attributes can be left out, except ``__version__``. Nevertheless please fill out most information -as you can, when you want to submit your plugin to the public repo. +An overview of acceptable values can be found in :class:`Base ` source code. +Unneeded attributes can be left out, except ``__version__``. Nevertheless please fill out most information +as you can, when you want to submit your plugin to the public repository. Additionally :class:`Crypter ` and :class:`Crypter ` -needs to have a specific regexp [2]_ ``__pattern__``. This will be matched against input urls and if a suited +needs to have a specific regexp [2]_ ``__pattern__``. This will be matched against input url's and if a suited plugin is found it is selected to handle the url. For localization pyLoad supports gettext [3]_, to mark strings for translation surround them with ``_("...")``. You don't need to subclass :class:`Base ` directly, but the -intermediate type according to your plugin. As example we choose an Hoster plugin, but the same is true for all +intermediate type according to your plugin. As an example we choose a hoster plugin, but the same is true for all plugin types. -How basic hoster plugin header could look like:: +How a basic hoster plugin header could look like:: from module.plugin.Hoster import Hoster class MyFileHoster(Hoster): __version__ = "0.1" __description__ = _("Short description of the plugin") - __long_description = _("""A even longer description - is not needed for hoster plugin, - but hook plugin should have it so the user knows what they doing.""") + __long_description = _("""An even longer description + is not needed for hoster plugins, + but the hook plugin should have it, so the users know what they are doing.""") In future examples the meta data will be left out, but remember it's required in every plugin! Config Entries -------------- -Every plugin is allowed to add entries to the config. These are defined via ``__config__`` and consists +Every plugin is allowed to add entries to the configuration. These are defined via ``__config__`` and consist of a list with tuples in the format of ``(name, type, verbose_name, default_value)`` or ``(name, type, verbose_name, short_description, default_value)``. @@ -56,7 +56,7 @@ Example from Youtube plugin:: (".mp4", "bool", _("Allow .mp4"), True)] -At runtime the desired config values can be retrieved with ``self.getConfig(name)`` and setted with +At runtime the desired config values can be retrieved with ``self.getConfig(name)`` and set with ``self.setConfig(name, value)``. Tagging Guidelines @@ -72,9 +72,9 @@ Keyword Meaning image Anything related to image(hoster) video Anything related to video(hoster) captcha A plugin that needs captcha decrypting -interaction A plugin that makes uses of interaction with user +interaction A plugin that makes use of interaction with the user free A hoster without any premium service -premium_only A hoster only useable with account +premium_only A hoster only usable with account ip_check A hoster that checks ip, that can be avoided with reconnect =============== =========================================================== @@ -89,7 +89,7 @@ and the :class:`Api ` at ``self.core.api`` With ``self.load(...)`` you can load any url and get the result. This method is only available to Hoster and Crypter. For other plugins use ``getURL(...)`` or ``getRequest()``. -Use ``self.store(...)`` and ``self.retrieve(...)`` to store data persistantly into the database. +Use ``self.store(...)`` and ``self.retrieve(...)`` to store data persistently into the database. Make use of ``logInfo, logError, logWarning, logDebug`` for logging purposes. @@ -98,15 +98,15 @@ Debugging One of the most important aspects in software programming is debugging. It is especially important for plugins which heavily rely on external input, which is true for all hoster and crypter plugins. -To enable debugging functionality start pyLoad with ``-d`` option or enable it in the config. +To enable debugging functionality start pyLoad with the ``-d`` option or enable it in the config. You should use ``self.logDebug(msg)`` when ever it is reasonable. It is a good pratice to log server output -or the calculation of results and then check in the log if it really it what you are expecting. +or the calculation of results and then check in the log if it really is what you are expecting. For further debugging you can install ipython [4]_, and set breakpoints with ``self.core.breakpoint()``. It will open the python debugger [5]_ and pause the plugin thread. To open a ipython shell in the running programm use ``self.shell()``. -These methods are usefull to gain access to the code flow at runtime and check or modify variables. +These methods are useful to gain access to the code flow at runtime and check or modify variables. .. rubric:: Footnotes diff --git a/docs/plugins/crypter_plugin.rst b/docs/plugins/crypter_plugin.rst index 4e7803808..8c54dccb1 100644 --- a/docs/plugins/crypter_plugin.rst +++ b/docs/plugins/crypter_plugin.rst @@ -4,7 +4,7 @@ Crypter - Extract links from pages ================================== We are starting with the simplest plugin, the :class:`Crypter `. -It's job is it to take a url as input and generate new package or links, for example by filtering the urls or +It's job is to take an url as input and generate a new package or links, for example by filtering the urls or loading a page and extracting links from the html code. You need to define the ``__pattern__`` to match target urls and subclass from :class:`Crypter `. :: @@ -36,11 +36,11 @@ create new Packages if needed by instantiating a :class:`Package` instance, whic html = self.load(url) - # .decrypt_from_content is only a example method here and will return a list of urls + # .decrypt_from_content is only an example method here and will return a list of urls urls = self.decrypt_from_content(html) return Package("my new package", urls) -And that's basically all you need to know. Just as little side-note if you want to use decrypter in +And that's basically all you need to know. Just as a little side-note if you want to use decrypter in your code you can use:: plugin = self.core.pluginManager.loadClass("crypter", "NameOfThePlugin") @@ -53,7 +53,7 @@ SimpleCrypter ------------- For simple crypter services there is the :class:`SimpleCrypter ` class which handles most of the workflow. Only the regexp -pattern have to be defined. +pattern has to be defined. Exmaple:: diff --git a/docs/plugins/hoster_plugin.rst b/docs/plugins/hoster_plugin.rst index ee112b570..b7546a313 100644 --- a/docs/plugins/hoster_plugin.rst +++ b/docs/plugins/hoster_plugin.rst @@ -24,13 +24,13 @@ An example ``process`` function could look like this :: # download the file, destination is determined by pyLoad self.download(parsed_url) -You need to know about the :class:`PyFile ` class, since an instance of it is given as parameter to every pyfile. -Some tasks your plugin should handle: proof if file is online, get filename, wait if needed, download the file, etc.. +You need to know about the :class:`PyFile ` class, since an instance of it is given as a parameter to every pyfile. +Some tasks your plugin should handle: check if the file is online, get filename, wait if needed, download the file, etc.. Wait times ---------- -Some hoster require you to wait a specific time. Just set the time with ``self.setWait(seconds)`` or +Some hosters require you to wait a specific time. Just set the time with ``self.setWait(seconds)`` or ``self.setWait(seconds, True)`` if you want pyLoad to perform a reconnect if needed. Captcha decrypting @@ -54,4 +54,4 @@ Testing Examples -------- -Best examples are already existing plugins in :file:`module/plugins/`. \ No newline at end of file +The best examples are the already existing plugins in :file:`module/plugins/`. \ No newline at end of file diff --git a/docs/plugins/overview.rst b/docs/plugins/overview.rst index 70db5ac90..b3debb053 100755 --- a/docs/plugins/overview.rst +++ b/docs/plugins/overview.rst @@ -12,10 +12,10 @@ Extending pyLoad .. rubric:: Motivation -pyLoad offers an comfortable and powerful plugin system to make extending possible. With it you only need to have some -python knowledge and can just start right away writing your own plugins. This document gives you an overwiew about the -conceptual part. You should not left out the :doc:`Base ` part, since it contains basic functionality for all plugins types. -A class diagram visualizing the relationship can be find below [1]_ +pyLoad offers a comfortable and powerful plugin system to make extensions possible. With it you only need to have some +python knowledge and can just start right away writing your own plugins. This document gives you an overview about the +conceptual part. You should not leave out the :doc:`Base ` part, since it contains basic functionality for all plugin types. +A class diagram visualizing the relationship can be found below [1]_ .. rubric:: Contents -- cgit v1.2.3 From e8eaa91da9e1236d8009df0d79bebe023de8933f Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 6 May 2012 13:04:15 +0200 Subject: little documentation update --- docs/api/datatypes.rst | 437 +++++++++++++++++++++------------ docs/api/json_api.rst | 36 ++- docs/api/overview.rst | 2 +- docs/module_overview.rst | 4 +- docs/plugins/account_plugin.rst | 6 + docs/plugins/base_plugin.rst | 6 +- docs/plugins/hoster_plugin.rst | 14 +- docs/plugins/overview.rst | 1 - docs/system/plugin_hierarchy.rst | 6 +- docs/system/pyload_DataLayout.png | Bin 0 -> 48354 bytes docs/system/pyload_PluginHierarchy.png | Bin 23553 -> 20713 bytes 11 files changed, 340 insertions(+), 172 deletions(-) create mode 100644 docs/system/pyload_DataLayout.png (limited to 'docs') diff --git a/docs/api/datatypes.rst b/docs/api/datatypes.rst index ad8c3df80..886d95a76 100644 --- a/docs/api/datatypes.rst +++ b/docs/api/datatypes.rst @@ -16,74 +16,96 @@ for various languages. It is also a good overview of avaible methods and return typedef i32 PackageID typedef i32 ResultID typedef i32 InteractionID + typedef i64 UTCDate + typedef i64 ByteCount typedef list LinkList + // a string that can represent multiple types int, bool, time, etc.. + typedef string ValueString typedef string PluginName - typedef byte Progress - typedef byte Priority - + // NA - Not Available enum DownloadStatus { - Finished + NA, Offline, Online, Queued, + Paused, + Finished, Skipped, + Failed, + Starting, Waiting, + Downloading, TempOffline, - Starting, - Failed, Aborted, Decrypting, - Custom, - Downloading, Processing, + Custom, Unknown } - enum Destination { - Collector, - Queue + enum MediaType { + All = 0 + Other = 1, + Audio = 2, + Image = 4, + Video = 8, + Document = 16, + Archive = 32, + } + + enum FileStatus { + Ok, + Missing, + Remote, // file is available at remote location + } + + enum PackageStatus { + Ok, + Paused, + Remote, } // types for user interaction // some may only be place holder currently not supported // also all input - output combination are not reasonable, see InteractionManager for further info enum Input { - NONE, - TEXT, - TEXTBOX, - PASSWORD, - BOOL, // confirm like, yes or no dialog - CLICK, // for positional captchas - CHOICE, // choice from list - MULTIPLE, // multiple choice from list of elements - LIST, // arbitary list of elements - TABLE // table like data structure + NA, + Text, + TextBox, + Password, + Bool, // confirm like, yes or no dialog + Click, // for positional captchas + Choice, // choice from list + Multiple, // multiple choice from list of elements + List, // arbitary list of elements + Table // table like data structure } // more can be implemented by need // this describes the type of the outgoing interaction // ensure they can be logcial or'ed enum Output { - CAPTCHA = 1, - QUESTION = 2, - NOTIFICATION = 4, + All = 0, + Notification = 1, + Captcha = 2, + Query = 4, } - struct DownloadInfo { + struct ProgressInfo { 1: FileID fid, 2: string name, - 3: i64 speed, + 3: ByteCount speed, 4: i32 eta, 5: string format_eta, - 6: i64 bleft, - 7: i64 size, + 6: ByteCount bleft, + 7: ByteCount size, 8: string format_size, - 9: Progress percent, + 9: i16 percent, 10: DownloadStatus status, 11: string statusmsg, 12: string format_wait, - 13: i64 wait_until, + 13: UTCDate wait_until, 14: PackageID packageID, 15: string packageName, 16: PluginName plugin, @@ -94,76 +116,107 @@ for various languages. It is also a good overview of avaible methods and return 2: i16 active, 3: i16 queue, 4: i16 total, - 5: i64 speed, + 5: ByteCount speed, 6: bool download, 7: bool reconnect } - struct FileData { + // download info for specific file + struct DownloadInfo { + 1: string url, + 2: PluginName plugin, + 3: string hash, + 4: DownloadStatus status, + 5: string statusmsg, + 6: string error, + } + + struct FileInfo { 1: FileID fid, - 2: string url, - 3: string name, - 4: PluginName plugin, - 5: i64 size, - 6: string format_size, - 7: DownloadStatus status, - 8: string statusmsg, - 9: PackageID packageID, - 10: string error, - 11: i16 order + 2: string name, + 3: PackageID package, + 4: ByteCount size, + 5: FileStatus status, + 6: MediaType media, + 7: UTCDate added, + 8: i16 fileorder, + 9: optional DownloadInfo download, } - struct PackageData { + struct PackageStats { + 1: i16 linkstotal, + 2: i16 linksdone, + 3: ByteCount sizetotal, + 4: ByteCount sizedone, + } + + struct PackageInfo { 1: PackageID pid, 2: string name, 3: string folder, - 4: string site, - 5: string password, - 6: Destination dest, - 7: i16 order, - 8: optional i16 linksdone, - 9: optional i64 sizedone, - 10: optional i64 sizetotal, - 11: optional i16 linkstotal, - 12: optional list links, - 13: optional list fids + 4: PackageID root, + 5: string site, + 6: string comment, + 7: string password, + 8: UTCDate added, + 9: PackageStatus status, + 10: i16 packageorder, + 11: PackageStats stats, + 12: list fids, + 13: list pids, + } + + // thrift does not allow recursive datatypes, so all data is accumulated and mapped with id + struct PackageView { + 1: PackageInfo root, + 2: map files, + 3: map packages + } + + // general info about link, used for collector and online results + struct LinkStatus { + 1: string url, + 2: string name, + 3: PluginName plugin, + 4: ByteCount size, // size <= 0 : unknown + 5: DownloadStatus status, + 6: string packagename, } struct InteractionTask { 1: InteractionID iid, 2: Input input, - 3: list structure, - 4: list preset, - 5: Output output, - 6: list data, - 7: string title, - 8: string description, - 9: string plugin, + 3: list data, + 4: Output output, + 5: optional ValueString default_value, + 6: string title, + 7: string description, + 8: PluginName plugin, + } + + struct AddonInfo { + 1: string func_name, + 2: string description, + 3: ValueString value, } struct ConfigItem { 1: string name, - 2: string long_name, + 2: string display_name, 3: string description, 4: string type, - 5: string default_value, - 6: string value, + 5: ValueString default_value, + 6: ValueString value, } struct ConfigSection { 1: string name, - 2: string long_name, + 2: string display_name, 3: string description, 4: string long_description, 5: optional list items, - 6: optional map handler, - } - - struct CaptchaTask { - 1: i16 tid, - 2: binary data, - 3: string type, - 4: string resultType + 6: optional list info, + 7: optional list handler, // if null plugin is not loaded } struct EventInfo { @@ -183,170 +236,242 @@ for various languages. It is also a good overview of avaible methods and return 1: PluginName plugin, 2: string loginname, 3: bool valid, - 4: i64 validuntil, - 5: i64 trafficleft, - 6: i64 maxtraffic, + 4: UTCDate validuntil, + 5: ByteCount trafficleft, + 6: ByteCount maxtraffic, 7: bool premium, 8: bool activated, 9: map options, } - struct ServiceCall { - 1: PluginName plugin, - 2: string func, - 3: string arguments, // empty string or json encoded list - } - - struct OnlineStatus { - 1: string name, - 2: PluginName plugin, - 3: string packagename, - 4: DownloadStatus status, - 5: i64 size, // size <= 0 : unknown + struct AddonService { + 1: string func_name, + 2: string description, + 3: optional i16 media, + 4: optional bool package, } struct OnlineCheck { - 1: ResultID rid, // -1 -> nothing more to get - 2: map data, //url to result + 1: ResultID rid, // -1 -> nothing more to get + 2: map data, //url to result } // exceptions - exception PackageDoesNotExists{ + exception PackageDoesNotExists { 1: PackageID pid } - exception FileDoesNotExists{ + exception FileDoesNotExists { 1: FileID fid } - exception UserDoesNotExists{ + exception UserDoesNotExists { 1: string user } - exception ServiceDoesNotExists{ + exception ServiceDoesNotExists { 1: string plugin 2: string func } - exception ServiceException{ + exception ServiceException { 1: string msg } service Pyload { - //config - string getConfigValue(1: string section, 2: string option), - void setConfigValue(1: string section, 2: string option, 3: string value), - map getConfig(), - map getPluginConfig(), - ConfigSection configureSection(1: string section), + /////////////////////// + // Server Status + /////////////////////// - // server status + string getServerVersion(), + ServerStatus statusServer(), void pauseServer(), void unpauseServer(), bool togglePause(), - ServerStatus statusServer(), - i64 freeSpace(), - string getServerVersion(), + ByteCount freeSpace(), void kill(), void restart(), list getLog(1: i32 offset), bool isTimeDownload(), bool isTimeReconnect(), bool toggleReconnect(), + void scanDownloadFolder(), - // download preparing + // downloads - information + list getProgressInfo(), + + /////////////////////// + // Configuration + /////////////////////// + + string getConfigValue(1: string section, 2: string option), + void setConfigValue(1: string section, 2: string option, 3: string value), + map getConfig(), + map getPluginConfig(), + ConfigSection configureSection(1: string section), + void setConfigHandler(1: PluginName plugin, 2: InteractionID iid, 3: ValueString value), + + /////////////////////// + // Download Preparing + /////////////////////// - // packagename - urls - map generatePackages(1: LinkList links), map checkURLs(1: LinkList urls), map parseURLs(1: string html, 2: string url), + // packagename - urls // parses results and generates packages OnlineCheck checkOnlineStatus(1: LinkList urls), OnlineCheck checkOnlineStatusContainer(1: LinkList urls, 2: string filename, 3: binary data) - // poll results from previosly started online check + // poll results from previously started online check OnlineCheck pollResults(1: ResultID rid), - // downloads - information - list statusDownloads(), - PackageData getPackageData(1: PackageID pid) throws (1: PackageDoesNotExists e), - PackageData getPackageInfo(1: PackageID pid) throws (1: PackageDoesNotExists e), - FileData getFileData(1: FileID fid) throws (1: FileDoesNotExists e), - list getQueue(), - list getCollector(), - list getQueueData(), - list getCollectorData(), - map getPackageOrder(1: Destination destination), - map getFileOrder(1: PackageID pid) - - // downloads - adding/deleting - list generateAndAddPackages(1: LinkList links, 2: Destination dest), - PackageID addPackage(1: string name, 2: LinkList links, 3: Destination dest, 4: string password), + map generatePackages(1: LinkList links), + + /////////////////////// + // Adding/Deleting + /////////////////////// + + list generateAndAddPackages(1: LinkList links, 2: bool paused), + list autoAddLinks(1: LinkList links), + + PackageID createPackage(1: string name, 2: string folder, 3: PackageID root, 4: string password, + 5: string site, 6: string comment, 7: bool paused), + + PackageID addPackage(1: string name, 2: LinkList links, 3: string password), + // same as above with paused attribute + PackageID addPackageP(1: string name, 2: LinkList links, 3: string password, 4: bool paused), + + // pid -1 is toplevel + PackageID addPackageChild(1: string name, 2: LinkList links, 3: string password, 4: PackageID root, 5: bool paused), + PackageID uploadContainer(1: string filename, 2: binary data), - void addFiles(1: PackageID pid, 2: LinkList links), + + void addLinks(1: PackageID pid, 2: LinkList links) throws (1: PackageDoesNotExists e), + + // these are real file operations and WILL delete files on disk void deleteFiles(1: list fids), void deletePackages(1: list pids), - // downloads - modifying - void pushToQueue(1: PackageID pid), - void pullFromQueue(1: PackageID pid), + /////////////////////// + // Collector + /////////////////////// + + list getCollector(), + + void addToCollector(1: LinkList links), + PackageID addFromCollector(1: string name, 2: bool paused), + void renameCollPack(1: string name, 2: string new_name), + void deleteCollPack(1: string name), + void deleteCollLink(1: string url), + + //////////////////////////// + // File Information retrival + //////////////////////////// + + PackageView getAllFiles(), + PackageView getAllUnfinishedFiles(), + + // pid -1 for root, full=False only delivers first level in tree + PackageView getFileTree(1: PackageID pid, 2: bool full), + PackageView getUnfinishedFileTree(1: PackageID pid, 2: bool full), + + // same as above with full=False + PackageView getPackageContent(1: PackageID pid), + + PackageInfo getPackageInfo(1: PackageID pid) throws (1: PackageDoesNotExists e), + FileInfo getFileInfo(1: FileID fid) throws (1: FileDoesNotExists e), + map findFiles(1: string pattern), + + /////////////////////// + // Modify Downloads + /////////////////////// + void restartPackage(1: PackageID pid), void restartFile(1: FileID fid), void recheckPackage(1: PackageID pid), - void stopAllDownloads(), void stopDownloads(1: list fids), - void setPackageName(1: PackageID pid, 2: string name), - void movePackage(1: Destination destination, 2: PackageID pid), - void moveFiles(1: list fids, 2: PackageID pid), - void orderPackage(1: PackageID pid, 2: i16 position), - void orderFile(1: FileID fid, 2: i16 position), - void setPackageData(1: PackageID pid, 2: map data) throws (1: PackageDoesNotExists e), - list deleteFinished(), + void stopAllDownloads(), void restartFailed(), - //events - list getEvents(1: string uuid) + ///////////////////////// + // Modify Files/Packages + ///////////////////////// + + void setFilePaused(1: FileID fid, 2: bool paused) throws (1: FileDoesNotExists e), + + // moving package while downloading is not possible, so they will return bool to indicate success + void setPackagePaused(1: PackageID pid, 2: bool paused) throws (1: PackageDoesNotExists e), + bool setPackageFolder(1: PackageID pid, 2: string path) throws (1: PackageDoesNotExists e), + void setPackageData(1: PackageID pid, 2: map data) throws (1: PackageDoesNotExists e), + + // as above, this will move files on disk + bool movePackage(1: PackageID pid, 2: PackageID root) throws (1: PackageDoesNotExists e), + bool moveFiles(1: list fids, 2: PackageID pid) throws (1: PackageDoesNotExists e), + + void orderPackage(1: list pids, 2: i16 position), + void orderFiles(1: list fids, 2: PackageID pid, 3: i16 position), + + /////////////////////// + // User Interaction + /////////////////////// + + // mode = Output types binary ORed + bool isInteractionWaiting(1: i16 mode), + InteractionTask getInteractionTask(1: i16 mode), + void setInteractionResult(1: InteractionID iid, 2: ValueString result), + + // generate a download link, everybody can download the file until timeout reached + string generateDownloadLink(1: FileID fid, 2: i16 timeout), + + list getNotifications(), + + map> getAddonHandler(), + void callAddonHandler(1: PluginName plugin, 2: string func, 3: PackageID pid_or_fid), + + /////////////////////// + // Event Handling + /////////////////////// + + list getEvents(1: string uuid), - //accounts + /////////////////////// + // Account Methods + /////////////////////// + list getAccounts(1: bool refresh), list getAccountTypes() void updateAccount(1: PluginName plugin, 2: string account, 3: string password, 4: map options), void removeAccount(1: PluginName plugin, 2: string account), - //auth + ///////////////////////// + // Auth+User Information + ///////////////////////// + bool login(1: string username, 2: string password), - UserData getUserData(1: string username, 2:string password) throws (1: UserDoesNotExists ex), + UserData getUserData(1: string username, 2: string password) throws (1: UserDoesNotExists ex), map getAllUserData(), - //services + /////////////////////// + // Addon Methods + /////////////////////// - // servicename : description - map> getServices(), + map> getServices(), bool hasService(1: PluginName plugin, 2: string func), - string call(1: ServiceCall info) throws (1: ServiceDoesNotExists ex, 2: ServiceException e), + // empty string or json encoded list as args + string call(1: PluginName plugin, 2: string func, 3: string arguments) throws (1: ServiceDoesNotExists ex, 2: ServiceException e), - //info - // {plugin: {name: value}} - map> getAllInfo(), - map getInfoByPlugin(1: PluginName plugin), + map> getAllInfo(), + list getInfoByPlugin(1: PluginName plugin), //scheduler // TODO - - // User interaction - - //captcha - bool isCaptchaWaiting(), - CaptchaTask getCaptchaTask(1: bool exclusive), - string getCaptchaTaskStatus(1: InteractionID tid), - void setCaptchaResult(1: InteractionID tid, 2: string result), } .. [[[end]]] diff --git a/docs/api/json_api.rst b/docs/api/json_api.rst index 6d55b13b4..bcf546dc0 100644 --- a/docs/api/json_api.rst +++ b/docs/api/json_api.rst @@ -66,7 +66,41 @@ Just use a JSON encoder before using them in the HTTP request. Please note that the data has to be urlencoded at last. (Most libraries will do that automatically) +Example +------- + +Here is a little python script that is able to send commands to the pyload api:: + + #!/usr/bin/env python + # -*- coding: utf-8 -*- + + from urllib import urlopen, urlencode + from json import dumps + + URL = "http://localhost:8001/api/%s" + + def login(user, pw): + params = {"username": user, "password": pw} + ret = urlopen(URL % "login", urlencode(params)) + return ret.read().strip("\"") + + # send arbitrary command to pyload api, parameter as keyword argument + def send(session, command, **kwargs): + # convert arguments to json format + params = dict([(k, dumps(v)) for k,v in kwargs.iteritems()]) + params["session"] = session + ret = urlopen(URL % command, urlencode(params)) + return ret.read() + + if __name__ == "__main__": + session = login("User", "pwhere") + print "Session id:", session + + result = send(session, "setCaptchaResult", tid=0, result="some string") + print result + + .. rubric:: Footnotes -.. [1] http://de.wikipedia.org/wiki/JavaScript_Object_Notation \ No newline at end of file +.. [1] http://de.wikipedia.org/wiki/JavaScript_Object_Notation diff --git a/docs/api/overview.rst b/docs/api/overview.rst index 37aa8a526..605cf4c97 100644 --- a/docs/api/overview.rst +++ b/docs/api/overview.rst @@ -13,7 +13,7 @@ From Wikipedia, the free encyclopedia [1]_: .. rubric:: Motivation The idea of the centralized pyLoad :class:`Api ` is to give uniform access to all integral parts -and plugins in pyLoad, and furthermore to other clients, written in arbitrary programming languages. +and plugins in pyLoad to other clients written in arbitrary programming languages. Most of the :class:`Api ` functionality is exposed via RPC [2]_ and accessible via thrift [3]_ or simple JSON objects [4]_. In conclusion the :class:`Api ` is accessible via many programming language, over network from remote machines and over browser with javascript. diff --git a/docs/module_overview.rst b/docs/module_overview.rst index 1e771f36a..b2b98313b 100644 --- a/docs/module_overview.rst +++ b/docs/module_overview.rst @@ -13,10 +13,10 @@ A little selection of most important modules in pyLoad. module.plugins.internal.SimpleHoster.SimpleHoster module.plugins.Crypter.Crypter module.plugins.internal.SimpleCrypter.SimpleCrypter - module.plugins.Hook.Hook + module.plugins.Addon.Addon module.plugins.Account.Account module.plugins.MultiHoster.MultiHoster - module.HookManager.HookManager + module.AddonManager.AddonManager module.interaction.EventManager.EventManager module.interaction.InteractionManager.InteractionManager module.interaction.InteractionTask.InteractionTask diff --git a/docs/plugins/account_plugin.rst b/docs/plugins/account_plugin.rst index 75bf61a75..e683f1604 100644 --- a/docs/plugins/account_plugin.rst +++ b/docs/plugins/account_plugin.rst @@ -3,3 +3,9 @@ Account - Premium Access ======================== +Example +------- + +MultiHoster +----------- + diff --git a/docs/plugins/base_plugin.rst b/docs/plugins/base_plugin.rst index 91a6eef44..f6813cf40 100644 --- a/docs/plugins/base_plugin.rst +++ b/docs/plugins/base_plugin.rst @@ -66,9 +66,9 @@ To categorize a plugin, a list of keywords can be assigned via ``__tags__`` attr tags as you like, but please look at this table first to choose your tags. With standardised keywords we can generate a better overview of the plugins and provide some search criteria. -=============== =========================================================== +=============== ================================================================= Keyword Meaning -=============== =========================================================== +=============== ================================================================= image Anything related to image(hoster) video Anything related to video(hoster) captcha A plugin that needs captcha decrypting @@ -76,7 +76,7 @@ interaction A plugin that makes use of interaction with the user free A hoster without any premium service premium_only A hoster only usable with account ip_check A hoster that checks ip, that can be avoided with reconnect -=============== =========================================================== +=============== ================================================================= Basic Methods ------------- diff --git a/docs/plugins/hoster_plugin.rst b/docs/plugins/hoster_plugin.rst index b7546a313..9cd99a1f5 100644 --- a/docs/plugins/hoster_plugin.rst +++ b/docs/plugins/hoster_plugin.rst @@ -13,6 +13,9 @@ An example ``process`` function could look like this :: """ plugin code """ + + def setup(): + #TODO def process(self, pyfile): html = self.load(pyfile.url) # load the content of the orginal pyfile.url to html @@ -27,21 +30,18 @@ An example ``process`` function could look like this :: You need to know about the :class:`PyFile ` class, since an instance of it is given as a parameter to every pyfile. Some tasks your plugin should handle: check if the file is online, get filename, wait if needed, download the file, etc.. -Wait times +Common Tasks ---------- Some hosters require you to wait a specific time. Just set the time with ``self.setWait(seconds)`` or ``self.setWait(seconds, True)`` if you want pyLoad to perform a reconnect if needed. -Captcha decrypting ------------------- - To handle captcha input just use ``self.decryptCaptcha(url, ...)``, it will be send to clients -or handled by :class:`Hook ` plugins +or handled by :class:`Addon ` plugins -User interaction ----------------- +Online status fetching +---------------------- SimpleHoster ------------ diff --git a/docs/plugins/overview.rst b/docs/plugins/overview.rst index b3debb053..bbea86756 100755 --- a/docs/plugins/overview.rst +++ b/docs/plugins/overview.rst @@ -28,7 +28,6 @@ A class diagram visualizing the relationship can be found below [1]_ addon_plugin.rst - .. rubric:: Footnotes .. [1] :ref:`plugin_hierarchy` \ No newline at end of file diff --git a/docs/system/plugin_hierarchy.rst b/docs/system/plugin_hierarchy.rst index b663943a3..0e10664c0 100644 --- a/docs/system/plugin_hierarchy.rst +++ b/docs/system/plugin_hierarchy.rst @@ -6,4 +6,8 @@ Plugin Hierarchy Class diagram that describes plugin relationships. -.. image:: pyload_PluginHierarchy.png \ No newline at end of file +.. image:: pyload_PluginHierarchy.png + +Class diagram showing the relationship of api/thrift datatypes. + +.. image:: pyload_DataLayout.png \ No newline at end of file diff --git a/docs/system/pyload_DataLayout.png b/docs/system/pyload_DataLayout.png new file mode 100644 index 000000000..f620e62c6 Binary files /dev/null and b/docs/system/pyload_DataLayout.png differ diff --git a/docs/system/pyload_PluginHierarchy.png b/docs/system/pyload_PluginHierarchy.png index f1a753ee2..df8526c6a 100644 Binary files a/docs/system/pyload_PluginHierarchy.png and b/docs/system/pyload_PluginHierarchy.png differ -- cgit v1.2.3 From 3d7ed0c367cf521b61ec1f78784dec73a3b7c32a Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 6 May 2012 20:40:39 +0200 Subject: some classes for multi user mode --- docs/system/pyload_PluginHierarchy.png | Bin 20713 -> 22550 bytes 1 file changed, 0 insertions(+), 0 deletions(-) (limited to 'docs') diff --git a/docs/system/pyload_PluginHierarchy.png b/docs/system/pyload_PluginHierarchy.png index df8526c6a..118d3a7a8 100644 Binary files a/docs/system/pyload_PluginHierarchy.png and b/docs/system/pyload_PluginHierarchy.png differ -- cgit v1.2.3 From 829244a6140763712d50ed046c33f415f2b04301 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 15 May 2012 19:22:34 +0200 Subject: some multiuser db changes --- docs/system/pyload_DataLayout.png | Bin 48354 -> 57771 bytes 1 file changed, 0 insertions(+), 0 deletions(-) (limited to 'docs') diff --git a/docs/system/pyload_DataLayout.png b/docs/system/pyload_DataLayout.png index f620e62c6..98ab31a69 100644 Binary files a/docs/system/pyload_DataLayout.png and b/docs/system/pyload_DataLayout.png differ -- cgit v1.2.3