MultiHoster plugin type, some fixes, new documentation structure

13 files changed, 477 insertions, 36 deletions

diff --git a/docs/access_api.rst b/docs/api/access_api.rst
index df69da8b2..efa1ae3fc 100644
--- a/docs/access_api.rst
+++ b/docs/api/access_api.rst
@@ -26,8 +26,9 @@ Using Thrift
 ------------
 
 Every thrift service has to define all data structures and declare every method which should be usable via rpc.
-This file is located :file:`module/remote/thriftbackend/pyload.thrift`, its very helpful to inform about
+This file is located at :file:`module/remote/thriftbackend/pyload.thrift`, its very helpful to inform about
 arguments and detailed structure of return types. However it does not contain any information about what the functions does.
+You can also look at it :doc:`here <datatypes>`
 
 Assuming you want to use the API in any other language than python than check if it is
 supported here http://wiki.apache.org/thrift/LibraryFeatures?action=show&redirect=LanguageSupport.
diff --git a/docs/api/datatypes.rst b/docs/api/datatypes.rst
new file mode 100644
index 000000000..02e3209cc
--- /dev/null
+++ b/docs/api/datatypes.rst
@@ -0,0 +1,352 @@
+.. _api_datatypes:
+
+***********************
+API Datatype Definition
+***********************
+
+Below you find a copy of :file:`module/remote/thriftbackend/pyload.thrift`, which is used to generate the data structs
+for various languages. It is also a good overview of avaible methods and return data.
+
+.. code-block:: c
+
+     .. [[[cog cog.out(open('module/remote/thriftbackend/pyload.thrift', 'rb').read()) ]]]
+     namespace java org.pyload.thrift
+
+     typedef i32 FileID
+     typedef i32 PackageID
+     typedef i32 ResultID
+     typedef i32 InteractionID
+     typedef list<string> LinkList
+     typedef string PluginName
+     typedef byte Progress
+     typedef byte Priority
+
+
+     enum DownloadStatus {
+       Finished
+       Offline,
+       Online,
+       Queued,
+       Skipped,
+       Waiting,
+       TempOffline,
+       Starting,
+       Failed,
+       Aborted,
+       Decrypting,
+       Custom,
+       Downloading,
+       Processing,
+       Unknown
+     }
+
+     enum Destination {
+       Collector,
+       Queue
+     }
+
+     // types for user interaction
+     // some may only be place holder currently not supported
+     // also all input - output combination are not reasonable, see InteractionManager for further info
+     enum Input {
+       NONE,
+       TEXT,
+       TEXTBOX,
+       PASSWORD,
+       BOOL,   // confirm like, yes or no dialog
+       CLICK,  // for positional captchas
+       CHOICE,  // choice from list
+       MULTIPLE,  // multiple choice from list of elements
+       LIST, // arbitary list of elements
+       TABLE  // table like data structure
+     }
+     // more can be implemented by need
+
+     // this describes the type of the outgoing interaction
+     // ensure they can be logcial or'ed
+     enum Output {
+       CAPTCHA = 1,
+       QUESTION = 2,
+       NOTIFICATION = 4,
+     }
+
+     struct DownloadInfo {
+       1: FileID fid,
+       2: string name,
+       3: i64 speed,
+       4: i32 eta,
+       5: string format_eta,
+       6: i64 bleft,
+       7: i64 size,
+       8: string format_size,
+       9: Progress percent,
+       10: DownloadStatus status,
+       11: string statusmsg,
+       12: string format_wait,
+       13: i64 wait_until,
+       14: PackageID packageID,
+       15: string packageName,
+       16: PluginName plugin,
+     }
+
+     struct ServerStatus {
+       1: bool pause,
+       2: i16 active,
+       3: i16 queue,
+       4: i16 total,
+       5: i64 speed,
+       6: bool download,
+       7: bool reconnect
+     }
+
+     struct FileData {
+       1: FileID fid,
+       2: string url,
+       3: string name,
+       4: PluginName plugin,
+       5: i64 size,
+       6: string format_size,
+       7: DownloadStatus status,
+       8: string statusmsg,
+       9: PackageID packageID,
+       10: string error,
+       11: i16 order
+     }
+
+     struct PackageData {
+       1: PackageID pid,
+       2: string name,
+       3: string folder,
+       4: string site,
+       5: string password,
+       6: Destination dest,
+       7: i16 order,
+       8: optional i16 linksdone,
+       9: optional i64 sizedone,
+       10: optional i64 sizetotal,
+       11: optional i16 linkstotal,
+       12: optional list<FileData> links,
+       13: optional list<FileID> fids
+     }
+
+     struct InteractionTask {
+       1: InteractionID iid,
+       2: Input input,
+       3: list<string> structure,
+       4: list<string> preset,
+       5: Output output,
+       6: list<string> data,
+       7: string title,
+       8: string description,
+       9: string plugin,
+     }
+
+     struct ConfigItem {
+       1: string name,
+       2: string long_name,
+       3: string description,
+       4: string type,
+       5: string default_value,
+       6: string value,
+     }
+
+     struct ConfigSection {
+       1: string name,
+       2: string long_name,
+       3: string description,
+       4: string long_description,
+       5: optional list<ConfigItem> items,
+       6: optional map<string, InteractionTask> handler,
+     }
+
+     struct CaptchaTask {
+       1: i16 tid,
+       2: binary data,
+       3: string type,
+       4: string resultType
+     }
+
+     struct EventInfo {
+       1: string eventname,
+       2: list<string> event_args,
+     }
+
+     struct UserData {
+       1: string name,
+       2: string email,
+       3: i32 role,
+       4: i32 permission,
+       5: string templateName
+     }
+
+     struct AccountInfo {
+       1: PluginName plugin,
+       2: string loginname,
+       3: bool valid,
+       4: i64 validuntil,
+       5: i64 trafficleft,
+       6: i64 maxtraffic,
+       7: bool premium,
+       8: bool activated,
+       9: map<string, string> options,
+     }
+
+     struct ServiceCall {
+         1: PluginName plugin,
+         2: string func,
+         3: string arguments, // empty string or json encoded list
+     }
+
+     struct OnlineStatus {
+         1: string name,
+         2: PluginName plugin,
+         3: string packagename,
+         4: DownloadStatus status,
+         5: i64 size,   // size <= 0 : unknown
+     }
+
+     struct OnlineCheck {
+         1: ResultID rid, // -1 -> nothing more to get
+         2: map<string, OnlineStatus> data, //url to result
+     }
+
+
+     // exceptions
+
+     exception PackageDoesNotExists{
+       1: PackageID pid
+     }
+
+     exception FileDoesNotExists{
+       1: FileID fid
+     }
+
+     exception UserDoesNotExists{
+       1: string user
+     }
+
+     exception ServiceDoesNotExists{
+       1: string plugin
+       2: string func
+     }
+
+     exception ServiceException{
+       1: string msg
+     }
+
+     service Pyload {
+
+       //config
+       string getConfigValue(1: string section, 2: string option),
+       void setConfigValue(1: string section, 2: string option, 3: string value),
+       map<string, ConfigSection> getConfig(),
+       map<PluginName, ConfigSection> getPluginConfig(),
+       ConfigSection configureSection(1: string section),
+
+       // server status
+       void pauseServer(),
+       void unpauseServer(),
+       bool togglePause(),
+       ServerStatus statusServer(),
+       i64 freeSpace(),
+       string getServerVersion(),
+       void kill(),
+       void restart(),
+       list<string> getLog(1: i32 offset),
+       bool isTimeDownload(),
+       bool isTimeReconnect(),
+       bool toggleReconnect(),
+
+       // download preparing
+
+       // packagename - urls
+       map<string, LinkList> generatePackages(1: LinkList links),
+       map<PluginName, LinkList> checkURLs(1: LinkList urls),
+       map<PluginName, LinkList> parseURLs(1: string html, 2: string url),
+
+       // parses results and generates packages
+       OnlineCheck checkOnlineStatus(1: LinkList urls),
+       OnlineCheck checkOnlineStatusContainer(1: LinkList urls, 2: string filename, 3: binary data)
+
+       // poll results from previosly started online check
+       OnlineCheck pollResults(1: ResultID rid),
+
+       // downloads - information
+       list<DownloadInfo> statusDownloads(),
+       PackageData getPackageData(1: PackageID pid) throws (1: PackageDoesNotExists e),
+       PackageData getPackageInfo(1: PackageID pid) throws (1: PackageDoesNotExists e),
+       FileData getFileData(1: FileID fid) throws (1: FileDoesNotExists e),
+       list<PackageData> getQueue(),
+       list<PackageData> getCollector(),
+       list<PackageData> getQueueData(),
+       list<PackageData> getCollectorData(),
+       map<i16, PackageID> getPackageOrder(1: Destination destination),
+       map<i16, FileID> getFileOrder(1: PackageID pid)
+
+       // downloads - adding/deleting
+       list<PackageID> generateAndAddPackages(1: LinkList links, 2: Destination dest),
+       PackageID addPackage(1: string name, 2: LinkList links, 3: Destination dest, 4: string password),
+       void addFiles(1: PackageID pid, 2: LinkList links),
+       void uploadContainer(1: string filename, 2: binary data),
+       void deleteFiles(1: list<FileID> fids),
+       void deletePackages(1: list<PackageID> pids),
+
+       // downloads - modifying
+       void pushToQueue(1: PackageID pid),
+       void pullFromQueue(1: PackageID pid),
+       void restartPackage(1: PackageID pid),
+       void restartFile(1: FileID fid),
+       void recheckPackage(1: PackageID pid),
+       void stopAllDownloads(),
+       void stopDownloads(1: list<FileID> fids),
+       void setPackageName(1: PackageID pid, 2: string name),
+       void movePackage(1: Destination destination, 2: PackageID pid),
+       void moveFiles(1: list<FileID> fids, 2: PackageID pid),
+       void orderPackage(1: PackageID pid, 2: i16 position),
+       void orderFile(1: FileID fid, 2: i16 position),
+       void setPackageData(1: PackageID pid, 2: map<string, string> data) throws (1: PackageDoesNotExists e),
+       list<PackageID> deleteFinished(),
+       void restartFailed(),
+
+       //events
+       list<EventInfo> getEvents(1: string uuid)
+       
+       //accounts
+       list<AccountInfo> getAccounts(1: bool refresh),
+       list<string> getAccountTypes()
+       void updateAccount(1: PluginName plugin, 2: string account, 3: string password, 4: map<string, string> options),
+       void removeAccount(1: PluginName plugin, 2: string account),
+       
+       //auth
+       bool login(1: string username, 2: string password),
+       UserData getUserData(1: string username, 2:string password) throws (1: UserDoesNotExists ex),
+       map<string, UserData> getAllUserData(),
+
+       //services
+
+       // servicename : description
+       map<PluginName, map<string, string>> getServices(),
+       bool hasService(1: PluginName plugin, 2: string func),
+       string call(1: ServiceCall info) throws (1: ServiceDoesNotExists ex, 2: ServiceException e),
+
+
+       //info
+       // {plugin: {name: value}}
+       map<PluginName, map<string,string>> getAllInfo(),
+       map<string, string> getInfoByPlugin(1: PluginName plugin),
+
+       //scheduler
+
+       // TODO
+
+
+       // User interaction
+
+       //captcha
+       bool isCaptchaWaiting(),
+       CaptchaTask getCaptchaTask(1: bool exclusive),
+       string getCaptchaTaskStatus(1: InteractionID tid),
+       void setCaptchaResult(1: InteractionID tid, 2: string result),
+     }
+     .. [[[end]]]
+
diff --git a/docs/api/overview.rst b/docs/api/overview.rst
new file mode 100644
index 000000000..02cee3e0d
--- /dev/null
+++ b/docs/api/overview.rst
@@ -0,0 +1,35 @@
+.. _overview:
+
+=======================================
+API - Application Programming Interface
+=======================================
+
+From Wikipedia, the free encyclopedia [1]_:
+
+    An application programming interface (API) is a source code based specification intended to be used as an interface
+    by software components to communicate with each other. An API may include specifications for routines,
+    data structures, object classes, and variables.
+
+.. rubric:: Motivation
+
+The idea of the centralized pyLoad :class:`Api <module.Api.Api>` is to give uniform access to all integral parts
+and plugins in pyLoad, and furthermore to other clients, written in arbitrary programming languages.
+Most of the :class:`Api <module.Api.Api>` functionality is exposed via RPC [2]_ and accessible via thrift [3]_ or
+simple JSON objects [4]_. In conclusion the :class:`Api <module.Api.Api>` is accessible via many programming language,
+over network from remote maschines and over browser with javascript.
+
+
+.. rubric:: Contents
+
+.. toctree::
+
+    access_api.rst
+    datatypes.rst
+
+
+.. rubric:: Footnotes
+
+.. [1] http://en.wikipedia.org/wiki/Application_programming_interface
+.. [2] http://en.wikipedia.org/wiki/Remote_procedure_call
+.. [3] http://en.wikipedia.org/wiki/Thrift_(protocol)
+.. [4] http://en.wikipedia.org/wiki/Json
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index 09e4d0c1c..454ed5967 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -65,8 +65,8 @@ v = options.version.split(".")
 cog.outl("version = '%s'" % ".".join(v[:2]))
 cog.outl("release = '%s'" % ".".join(v))
 ]]]"""
-version = '0.4'
-release = '0.4.9'
+version = '0.5'
+release = '0.5.0'
 # [[[end]]]
 
 
diff --git a/docs/extend_pyload.rst b/docs/extend_pyload.rst
deleted file mode 100755
index 337cb6854..000000000
--- a/docs/extend_pyload.rst
+++ /dev/null
@@ -1,13 +0,0 @@
-.. _extend_pyload:
-
-********************
-How to extend pyLoad                                                                                                                
-********************
-
-In general there a two different plugin types. These allow everybody to write powerful, modular plugins without knowing
-every detail of the pyLoad core. However you should have some basic knowledge of python.
-
-.. toctree::
-
-    write_hooks.rst
-    write_plugins.rst
\ No newline at end of file
diff --git a/docs/index.rst b/docs/index.rst
index 757fd7537..befac0fd2 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -1,27 +1,46 @@
-.. pyLoad documentation master file, created by
-   sphinx-quickstart on Sat Jun  4 11:54:34 2011.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
+.. pyLoad documentation master file
 
-Welcome to pyLoad's documentation!
-==================================
+=====================
+pyLoad Documentation
+=====================
 
-Great that you found your way to the pyLoad documentation!
+.. image:: _static/logo.png
+    :height: 144
+    :width: 412
 
-We have collected some information here to help developer writing plugins and understandig our code.
-If you want to help us developing visit us in our IRC channel #pyload on freenode.net or leave a message in our forum.
 
-Contents:
+Great that you found your way to the pyLoad [1]_ documentation!
+
+This is the ultimate document to get started extending or accessing pyLoad in your own way.
+We will cover on how to access the API so you can write your own client to pyLoad. The next big part gives you an idea
+how to extend pyLoad and write your own powerful plugins, which perfectly integrate into our system.
+
+The complete pyLoad source and this documentation is available at bitbucket [2]_. If you would like to contribute
+come around in our irc channel [3]_ or open a pull request.
+In case you still have question, ask them at our forum [4]_ or in our official irc channel at #pyload @ irc.freenode.net
+
+We wish you the best of luck and happy programming.
+-- the pyLoad Team
+
+Contents
+--------
 
 .. toctree::
    :maxdepth: 2
 
-   access_api.rst
-   extend_pyload.rst
+   api/overview.rst
+   plugins/overview.rst
    module_overview.rst
 
 .. currentmodule:: module
 
+.. rubric:: Footnotes
+
+.. [1] http://pyload.org
+.. [2] http://pyload.org/irc
+.. [3] http://bitbucket.org/spoob/pyload/overview
+.. [4] http://forum.pyload.org
+
 ==================
 
 * :ref:`genindex`
diff --git a/docs/module_overview.rst b/docs/module_overview.rst
index 309dccdfd..b8db51538 100644
--- a/docs/module_overview.rst
+++ b/docs/module_overview.rst
@@ -1,7 +1,8 @@
+
 Module Overview
 ===============
 
-You can find an overview of some important classes here:
+A little selection of most important modules in pyLoad.
 
 .. autosummary::
    :toctree: module
@@ -10,8 +11,9 @@ You can find an overview of some important classes here:
    module.plugins.Base.Base
    module.plugins.Hoster.Hoster
    module.plugins.Crypter.Crypter
-   module.plugins.Account.Account
    module.plugins.Hook.Hook
+   module.plugins.Account.Account
+   module.plugins.MultiHoster.MultiHoster
    module.HookManager.HookManager
    module.interaction.EventManager.EventManager
    module.interaction.InteractionManager.InteractionManager
diff --git a/docs/plugins/account_plugin.rst b/docs/plugins/account_plugin.rst
new file mode 100644
index 000000000..75bf61a75
--- /dev/null
+++ b/docs/plugins/account_plugin.rst
@@ -0,0 +1,5 @@
+.. _account_plugin:
+
+Account - Premium Access
+========================
+
diff --git a/docs/plugins/base_plugin.rst b/docs/plugins/base_plugin.rst
new file mode 100644
index 000000000..62ab248ef
--- /dev/null
+++ b/docs/plugins/base_plugin.rst
@@ -0,0 +1,5 @@
+.. _base_plugin:
+
+Base Plugin - And here it begins...
+===================================
+
diff --git a/docs/plugins/crypter_plugin.rst b/docs/plugins/crypter_plugin.rst
new file mode 100644
index 000000000..d910ec412
--- /dev/null
+++ b/docs/plugins/crypter_plugin.rst
@@ -0,0 +1,5 @@
+.. _crypter_plugin:
+
+Crypter - Extract links from pages
+==================================
+
diff --git a/docs/write_hooks.rst b/docs/plugins/hook_plugin.rst
index dd60367b7..e263ece2e 100644
--- a/docs/write_hooks.rst
+++ b/docs/plugins/hook_plugin.rst
@@ -1,7 +1,7 @@
 .. _write_hooks:
 
-Hooks
-=====
+Hook - Do everything you want
+=============================
 
 A Hook is a python file which is located at :file:`module/plugins/hooks`.
 The :class:`HookManager <module.HookManager.HookManager>` will load it automatically on startup. Only one instance exists
diff --git a/docs/write_plugins.rst b/docs/plugins/hoster_plugin.rst
index b513a5978..59f35f5cb 100644
--- a/docs/write_plugins.rst
+++ b/docs/plugins/hoster_plugin.rst
@@ -1,7 +1,7 @@
-.. _write_plugins:
+.. _hoster_plugin:
 
-Plugins
-=======
+Hoster - Load files to disk
+===========================
 
 A Plugin is a python file located at one of the subfolders in :file:`module/plugins/`. Either :file:`hoster`, :file:`crypter`
 or :file:`container`, depending of it's type.
@@ -22,7 +22,6 @@ How basic hoster plugin header could look like: ::
         from module.plugin.Hoster import Hoster
 
         class MyFileHoster(Hoster):
-                __name__ = "MyFileHoster"
                 __version__ = "0.1"
                 __pattern__ = r"http://myfilehoster.example.com/file_id/[0-9]+"
                 __config__ = []
diff --git a/docs/plugins/overview.rst b/docs/plugins/overview.rst
new file mode 100755
index 000000000..23913b787
--- /dev/null
+++ b/docs/plugins/overview.rst
@@ -0,0 +1,31 @@
+.. _overview:
+
+================
+Extending pyLoad
+================
+
+.. pull-quote::
+    Any sufficiently advanced technology is indistinguishable from magic.
+
+    -- Arthur C. Clarke
+
+
+.. rubric:: Motivation
+
+pyLoad offers an comfortable and powerful plugin system to make extending possible. With it you only need to have some
+python knowledge and can just start right away writing your own plugins. This document gives you an overwiew about the
+conceptual part. You should not left out the `Base` part, since it contains basic functionality for all plugins types.
+
+.. rubric:: Contents
+
+.. toctree::
+
+    base_plugin.rst
+    crypter_plugin.rst
+    hoster_plugin.rst
+    account_plugin.rst
+    hook_plugin.rst
+
+
+
+.. rubric:: Footnotes
\ No newline at end of file