merged stable into default

4 files changed, 693 insertions, 0 deletions

diff --git a/docs/api/datatypes.rst b/docs/api/datatypes.rst
new file mode 100644
index 000000000..886d95a76
--- /dev/null
+++ b/docs/api/datatypes.rst
@@ -0,0 +1,477 @@
+.. _api_datatypes:
+
+***********************
+API Datatype Definition
+***********************
+
+Below you find a copy of :file:`module/remote/thriftbackend/pyload.thrift`, which is used to generate the data structs
+for various languages. It is also a good overview of avaible methods and return data.
+
+.. code-block:: c
+
+     .. [[[cog cog.out(open('module/remote/thriftbackend/pyload.thrift', 'rb').read()) ]]]
+     namespace java org.pyload.thrift
+
+     typedef i32 FileID
+     typedef i32 PackageID
+     typedef i32 ResultID
+     typedef i32 InteractionID
+     typedef i64 UTCDate
+     typedef i64 ByteCount
+     typedef list<string> LinkList
+     // a string that can represent multiple types int, bool, time, etc..
+     typedef string ValueString
+     typedef string PluginName
+
+     // NA - Not Available
+     enum DownloadStatus {
+       NA,
+       Offline,
+       Online,
+       Queued,
+       Paused,
+       Finished,
+       Skipped,
+       Failed,
+       Starting,
+       Waiting,
+       Downloading,
+       TempOffline,
+       Aborted,
+       Decrypting,
+       Processing,
+       Custom,
+       Unknown
+     }
+
+     enum MediaType {
+       All = 0
+       Other = 1,
+       Audio = 2,
+       Image = 4,
+       Video = 8,
+       Document = 16,
+       Archive = 32,
+     }
+
+     enum FileStatus {
+       Ok,
+       Missing,
+       Remote,   // file is available at remote location
+     }
+
+     enum PackageStatus {
+       Ok,
+       Paused,
+       Remote,
+     }
+
+     // types for user interaction
+     // some may only be place holder currently not supported
+     // also all input - output combination are not reasonable, see InteractionManager for further info
+     enum Input {
+       NA,
+       Text,
+       TextBox,
+       Password,
+       Bool,   // confirm like, yes or no dialog
+       Click,  // for positional captchas
+       Choice,  // choice from list
+       Multiple,  // multiple choice from list of elements
+       List, // arbitary list of elements
+       Table  // table like data structure
+     }
+     // more can be implemented by need
+
+     // this describes the type of the outgoing interaction
+     // ensure they can be logcial or'ed
+     enum Output {
+       All = 0,
+       Notification = 1,
+       Captcha = 2,
+       Query = 4,
+     }
+
+     struct ProgressInfo {
+       1: FileID fid,
+       2: string name,
+       3: ByteCount speed,
+       4: i32 eta,
+       5: string format_eta,
+       6: ByteCount bleft,
+       7: ByteCount size,
+       8: string format_size,
+       9: i16 percent,
+       10: DownloadStatus status,
+       11: string statusmsg,
+       12: string format_wait,
+       13: UTCDate wait_until,
+       14: PackageID packageID,
+       15: string packageName,
+       16: PluginName plugin,
+     }
+
+     struct ServerStatus {
+       1: bool pause,
+       2: i16 active,
+       3: i16 queue,
+       4: i16 total,
+       5: ByteCount speed,
+       6: bool download,
+       7: bool reconnect
+     }
+
+     // download info for specific file
+     struct DownloadInfo {
+       1: string url,
+       2: PluginName plugin,
+       3: string hash,
+       4: DownloadStatus status,
+       5: string statusmsg,
+       6: string error,
+     }
+
+     struct FileInfo {
+       1: FileID fid,
+       2: string name,
+       3: PackageID package,
+       4: ByteCount size,
+       5: FileStatus status,
+       6: MediaType media,
+       7: UTCDate added,
+       8: i16 fileorder,
+       9: optional DownloadInfo download,
+     }
+
+     struct PackageStats {
+       1: i16 linkstotal,
+       2: i16 linksdone,
+       3: ByteCount sizetotal,
+       4: ByteCount sizedone,
+     }
+
+     struct PackageInfo {
+       1: PackageID pid,
+       2: string name,
+       3: string folder,
+       4: PackageID root,
+       5: string site,
+       6: string comment,
+       7: string password,
+       8: UTCDate added,
+       9: PackageStatus status,
+       10: i16 packageorder,
+       11: PackageStats stats,
+       12: list<FileID> fids,
+       13: list<PackageID> pids,
+     }
+
+     // thrift does not allow recursive datatypes, so all data is accumulated and mapped with id
+     struct PackageView {
+       1: PackageInfo root,
+       2: map<FileID, FileInfo> files,
+       3: map<PackageID, PackageInfo> packages
+     }
+
+     // general info about link, used for collector and online results
+     struct LinkStatus {
+         1: string url,
+         2: string name,
+         3: PluginName plugin,
+         4: ByteCount size,   // size <= 0 : unknown
+         5: DownloadStatus status,
+         6: string packagename,
+     }
+
+     struct InteractionTask {
+       1: InteractionID iid,
+       2: Input input,
+       3: list<string> data,
+       4: Output output,
+       5: optional ValueString default_value,
+       6: string title,
+       7: string description,
+       8: PluginName plugin,
+     }
+
+     struct AddonInfo {
+       1: string func_name,
+       2: string description,
+       3: ValueString value,
+     }
+
+     struct ConfigItem {
+       1: string name,
+       2: string display_name,
+       3: string description,
+       4: string type,
+       5: ValueString default_value,
+       6: ValueString value,
+     }
+
+     struct ConfigSection {
+       1: string name,
+       2: string display_name,
+       3: string description,
+       4: string long_description,
+       5: optional list<ConfigItem> items,
+       6: optional list<AddonInfo> info,
+       7: optional list<InteractionTask> handler, // if null plugin is not loaded
+     }
+
+     struct EventInfo {
+       1: string eventname,
+       2: list<string> event_args,
+     }
+
+     struct UserData {
+       1: string name,
+       2: string email,
+       3: i32 role,
+       4: i32 permission,
+       5: string templateName
+     }
+
+     struct AccountInfo {
+       1: PluginName plugin,
+       2: string loginname,
+       3: bool valid,
+       4: UTCDate validuntil,
+       5: ByteCount trafficleft,
+       6: ByteCount maxtraffic,
+       7: bool premium,
+       8: bool activated,
+       9: map<string, string> options,
+     }
+
+     struct AddonService {
+       1: string func_name,
+       2: string description,
+       3: optional i16 media,
+       4: optional bool package,
+     }
+
+     struct OnlineCheck {
+       1: ResultID rid, // -1 -> nothing more to get
+       2: map<string, LinkStatus> data, //url to result
+     }
+
+
+     // exceptions
+
+     exception PackageDoesNotExists {
+       1: PackageID pid
+     }
+
+     exception FileDoesNotExists {
+       1: FileID fid
+     }
+
+     exception UserDoesNotExists {
+       1: string user
+     }
+
+     exception ServiceDoesNotExists {
+       1: string plugin
+       2: string func
+     }
+
+     exception ServiceException {
+       1: string msg
+     }
+
+     service Pyload {
+
+       ///////////////////////
+       // Server Status
+       ///////////////////////
+
+       string getServerVersion(),
+       ServerStatus statusServer(),
+       void pauseServer(),
+       void unpauseServer(),
+       bool togglePause(),
+       ByteCount freeSpace(),
+       void kill(),
+       void restart(),
+       list<string> getLog(1: i32 offset),
+       bool isTimeDownload(),
+       bool isTimeReconnect(),
+       bool toggleReconnect(),
+       void scanDownloadFolder(),
+
+       // downloads - information
+       list<ProgressInfo> getProgressInfo(),
+
+       ///////////////////////
+       // Configuration
+       ///////////////////////
+
+       string getConfigValue(1: string section, 2: string option),
+       void setConfigValue(1: string section, 2: string option, 3: string value),
+       map<string, ConfigSection> getConfig(),
+       map<PluginName, ConfigSection> getPluginConfig(),
+       ConfigSection configureSection(1: string section),
+       void setConfigHandler(1: PluginName plugin, 2: InteractionID iid, 3: ValueString value),
+
+       ///////////////////////
+       // Download Preparing
+       ///////////////////////
+
+       map<PluginName, LinkList> checkURLs(1: LinkList urls),
+       map<PluginName, LinkList> parseURLs(1: string html, 2: string url),
+       // packagename - urls
+
+       // parses results and generates packages
+       OnlineCheck checkOnlineStatus(1: LinkList urls),
+       OnlineCheck checkOnlineStatusContainer(1: LinkList urls, 2: string filename, 3: binary data)
+
+       // poll results from previously started online check
+       OnlineCheck pollResults(1: ResultID rid),
+
+       map<string, LinkList> generatePackages(1: LinkList links),
+
+       ///////////////////////
+       // Adding/Deleting
+       ///////////////////////
+
+       list<PackageID> generateAndAddPackages(1: LinkList links, 2: bool paused),
+       list<FileID> autoAddLinks(1: LinkList links),
+
+       PackageID createPackage(1: string name, 2: string folder, 3: PackageID root, 4: string password,
+                                 5: string site, 6: string comment, 7: bool paused),
+
+       PackageID addPackage(1: string name, 2: LinkList links, 3: string password),
+       // same as above with paused attribute
+       PackageID addPackageP(1: string name, 2: LinkList links, 3: string password, 4: bool paused),
+
+       // pid -1 is toplevel
+       PackageID addPackageChild(1: string name, 2: LinkList links, 3: string password, 4: PackageID root, 5: bool paused),
+
+       PackageID uploadContainer(1: string filename, 2: binary data),
+
+       void addLinks(1: PackageID pid, 2: LinkList links) throws (1: PackageDoesNotExists e),
+
+       // these are real file operations and WILL delete files on disk
+       void deleteFiles(1: list<FileID> fids),
+       void deletePackages(1: list<PackageID> pids),
+
+       ///////////////////////
+       // Collector
+       ///////////////////////
+
+       list<LinkStatus> getCollector(),
+
+       void addToCollector(1: LinkList links),
+       PackageID addFromCollector(1: string name, 2: bool paused),
+       void renameCollPack(1: string name, 2: string new_name),
+       void deleteCollPack(1: string name),
+       void deleteCollLink(1: string url),
+
+       ////////////////////////////
+       // File Information retrival
+       ////////////////////////////
+
+       PackageView getAllFiles(),
+       PackageView getAllUnfinishedFiles(),
+
+       // pid -1 for root, full=False only delivers first level in tree
+       PackageView getFileTree(1: PackageID pid, 2: bool full),
+       PackageView getUnfinishedFileTree(1: PackageID pid, 2: bool full),
+
+       // same as above with full=False
+       PackageView getPackageContent(1: PackageID pid),
+
+       PackageInfo getPackageInfo(1: PackageID pid) throws (1: PackageDoesNotExists e),
+       FileInfo getFileInfo(1: FileID fid) throws (1: FileDoesNotExists e),
+       map<FileID, FileInfo> findFiles(1: string pattern),
+
+       ///////////////////////
+       // Modify Downloads
+       ///////////////////////
+
+       void restartPackage(1: PackageID pid),
+       void restartFile(1: FileID fid),
+       void recheckPackage(1: PackageID pid),
+       void stopDownloads(1: list<FileID> fids),
+       void stopAllDownloads(),
+       void restartFailed(),
+
+       /////////////////////////
+       // Modify Files/Packages
+       /////////////////////////
+
+       void setFilePaused(1: FileID fid, 2: bool paused) throws (1: FileDoesNotExists e),
+
+       // moving package while downloading is not possible, so they will return bool to indicate success
+       void setPackagePaused(1: PackageID pid, 2: bool paused) throws (1: PackageDoesNotExists e),
+       bool setPackageFolder(1: PackageID pid, 2: string path) throws (1: PackageDoesNotExists e),
+       void setPackageData(1: PackageID pid, 2: map<string, string> data) throws (1: PackageDoesNotExists e),
+
+       // as above, this will move files on disk
+       bool movePackage(1: PackageID pid, 2: PackageID root) throws (1: PackageDoesNotExists e),
+       bool moveFiles(1: list<FileID> fids, 2: PackageID pid) throws (1: PackageDoesNotExists e),
+
+       void orderPackage(1: list<PackageID> pids, 2: i16 position),
+       void orderFiles(1: list<FileID> fids, 2: PackageID pid, 3: i16 position),
+
+       ///////////////////////
+       // User Interaction
+       ///////////////////////
+
+       // mode = Output types binary ORed
+       bool isInteractionWaiting(1: i16 mode),
+       InteractionTask getInteractionTask(1: i16 mode),
+       void setInteractionResult(1: InteractionID iid, 2: ValueString result),
+
+       // generate a download link, everybody can download the file until timeout reached
+       string generateDownloadLink(1: FileID fid, 2: i16 timeout),
+
+       list<InteractionTask> getNotifications(),
+
+       map<PluginName, list<AddonService>> getAddonHandler(),
+       void callAddonHandler(1: PluginName plugin, 2: string func, 3: PackageID pid_or_fid),
+
+       ///////////////////////
+       // Event Handling
+       ///////////////////////
+
+       list<EventInfo> getEvents(1: string uuid),
+       
+       ///////////////////////
+       // Account Methods
+       ///////////////////////
+
+       list<AccountInfo> getAccounts(1: bool refresh),
+       list<string> getAccountTypes()
+       void updateAccount(1: PluginName plugin, 2: string account, 3: string password, 4: map<string, string> options),
+       void removeAccount(1: PluginName plugin, 2: string account),
+       
+       /////////////////////////
+       // Auth+User Information
+       /////////////////////////
+
+       bool login(1: string username, 2: string password),
+       UserData getUserData(1: string username, 2: string password) throws (1: UserDoesNotExists ex),
+       map<string, UserData> getAllUserData(),
+
+       ///////////////////////
+       // Addon Methods
+       ///////////////////////
+
+       map<PluginName, list<AddonService>> getServices(),
+       bool hasService(1: PluginName plugin, 2: string func),
+
+       // empty string or json encoded list as args
+       string call(1: PluginName plugin, 2: string func, 3: string arguments) throws (1: ServiceDoesNotExists ex, 2: ServiceException e),
+
+       map<PluginName, list<AddonInfo>> getAllInfo(),
+       list<AddonInfo> getInfoByPlugin(1: PluginName plugin),
+
+       //scheduler
+
+       // TODO
+
+     }
+     .. [[[end]]]
+
diff --git a/docs/api/json_api.rst b/docs/api/json_api.rst
new file mode 100644
index 000000000..bcf546dc0
--- /dev/null
+++ b/docs/api/json_api.rst
@@ -0,0 +1,106 @@
+.. _json_api:
+
+========
+JSON API
+========
+
+JSON [1]_ is a lightweight object notation and wrappers exist for nearly every programming language. Every
+modern browser is able to load JSON objects with JavaScript. Unlike to thrift you don't need to generate or precompile
+any stub methods, the JSON :class:`Api <module.Api.Api>` is ready to be used in most languages. The library is really lightweight (at least in python)
+and you can build very lightweight scripts with it. Because of the builtin support, JSON is the first choice for all browser
+applications.
+
+In our case JSON is just the output format, you have exactly the same methods available as with the thrift backend. The only
+difference is the underlying protocol.
+
+Are there still reasons to choose the original :doc:`thrift <thrift_api>` backend in favor to JSON? Yes, since it
+uses a binary protocol the performance will be better (when generating the objects), traffic will be smaller and
+therefore the transfer faster.
+In most IDEs you will get code completion, because of the pre-generated classes, which can make work much easier.
+
+If you intend to write a full client you should prefer thrift if the language is supported, for lightweight scripts and
+in browser environments JSON will be the better choice.
+
+Login
+-----
+
+First you need to authenticate, if you are using this within the web interface and the user is logged in, the API is also accessible,
+since they share the same cookie/session.
+
+However, if you are building an external client and want to authenticate manually
+you have to send your credentials ``username`` and ``password`` as
+POST parameter to ``http://pyload-core/api/login``.
+
+The result will be your session id. If you are using cookies, it will be set and you can use the API now.
+In case you don't have cookies enabled you can pass the session id as ``session`` POST parameter
+so pyLoad can authenticate you.
+
+
+Calling Methods
+---------------
+
+In general you can use any method listed at the :class:`Api <module.Api.Api>` documentation, which is also available to
+the thrift backend.
+
+Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address
+or hostname including the web interface port. By default on local access this would be `localhost:8000`.
+
+The return value will be formatted in JSON, complex data types as dictionaries. Definition for data types can be found
+:doc:`here <datatypes>`
+
+Passing parameters
+------------------
+
+To pass arguments you have two choices:
+Either use positional arguments, e.g.: ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword
+arguments supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names
+in the :class:`Api <module.Api.Api>` documentation.
+
+It is important that *all* arguments are in JSON format. So ``http://pyload-core/api/getFileData/1`` is valid because
+1 represents an integer in json format. On the other hand if the method is expecting strings, this would be correct:
+``http://pyload-core/api/getUserData/"username"/"password"``.
+
+Strings are wrapped in double qoutes, because `"username"` represents a string in JSON format. It's not limited to
+strings and integers, every container type like lists and dicts are possible. You usually don't have to convert them.
+Just use a JSON encoder before using them in the HTTP request.
+
+Please note that the data has to be urlencoded at last. (Most libraries will do that automatically)
+
+Example
+-------
+
+Here is a little python script that is able to send commands to the pyload api::
+
+    #!/usr/bin/env python
+    # -*- coding: utf-8 -*-
+
+    from urllib import urlopen, urlencode
+    from json import dumps
+
+    URL = "http://localhost:8001/api/%s"
+
+    def login(user, pw):
+        params = {"username": user, "password": pw}
+        ret = urlopen(URL % "login", urlencode(params))
+        return ret.read().strip("\"")
+
+    # send arbitrary command to pyload api, parameter as keyword argument
+    def send(session, command, **kwargs):
+        # convert arguments to json format
+        params = dict([(k, dumps(v)) for k,v in kwargs.iteritems()])
+        params["session"] = session
+        ret = urlopen(URL % command, urlencode(params))
+        return ret.read()
+
+    if __name__ == "__main__":
+        session = login("User", "pwhere")
+        print "Session id:", session
+
+        result = send(session, "setCaptchaResult", tid=0, result="some string")
+        print result
+
+
+
+.. rubric:: Footnotes
+
+.. [1] http://de.wikipedia.org/wiki/JavaScript_Object_Notation
diff --git a/docs/api/overview.rst b/docs/api/overview.rst
new file mode 100644
index 000000000..605cf4c97
--- /dev/null
+++ b/docs/api/overview.rst
@@ -0,0 +1,36 @@
+.. _overview:
+
+=======================================
+API - Application Programming Interface
+=======================================
+
+From Wikipedia, the free encyclopedia [1]_:
+
+    An application programming interface (API) is a source code based specification intended to be used as an interface
+    by software components to communicate with each other. An API may include specifications for routines,
+    data structures, object classes, and variables.
+
+.. rubric:: Motivation
+
+The idea of the centralized pyLoad :class:`Api <module.Api.Api>` is to give uniform access to all integral parts
+and plugins in pyLoad  to other clients written in arbitrary programming languages.
+Most of the :class:`Api <module.Api.Api>` functionality is exposed via RPC [2]_ and accessible via thrift [3]_ or
+simple JSON objects [4]_. In conclusion the :class:`Api <module.Api.Api>` is accessible via many programming language,
+over network from remote machines and over browser with javascript.
+
+
+.. rubric:: Contents
+
+.. toctree::
+
+    thrift_api.rst
+    json_api.rst
+    datatypes.rst
+
+
+.. rubric:: Footnotes
+
+.. [1] http://en.wikipedia.org/wiki/Application_programming_interface
+.. [2] http://en.wikipedia.org/wiki/Remote_procedure_call
+.. [3] `<http://en.wikipedia.org/wiki/Thrift_(protocol)>`_
+.. [4] http://en.wikipedia.org/wiki/Json
\ No newline at end of file
diff --git a/docs/api/thrift_api.rst b/docs/api/thrift_api.rst
new file mode 100644
index 000000000..cd1d2f23c
--- /dev/null
+++ b/docs/api/thrift_api.rst
@@ -0,0 +1,74 @@
+.. _thrift_api:
+
+==========
+Thrift API
+==========
+
+Thrift [1]_ was first developed in-house at facebook, but later published to public domain and developed at Apache Incubator.
+It includes a binary protocol for remote calls, which has a much better performance than other data formats like XML, additionally
+it is available for numerous languages and therefore we choose it as primary backend for our API.
+
+First of all, you need to know what you can do with our API. It lets you do all common task like
+retrieving download status, manage queue, manage accounts, modify config and so on.
+
+This document is not intended to explain every function in detail, for a complete listing
+see :class:`Api <module.Api.Api>`.
+
+Of course its possible to access the ``core.api`` attribute in plugins and hooks, but much more
+interesting is the possibility to call function from different programs written in many different languages.
+
+pyLoad uses thrift as backend and provides its :class:`Api <module.Api.Api>` as service.
+More information about thrift can be found in their  wiki [2]_.
+
+
+Using Thrift
+------------
+
+Every thrift service has to define all data structures and declare every method which should be usable via rpc.
+This file is located at :file:`module/remote/thriftbackend/pyload.thrift`, its very helpful to inform about
+arguments and detailed structure of return types. However it does not contain any information about what the functions does.
+You can also look at it :doc:`here <datatypes>`
+
+Assuming you want to use the API in any other language than python than check if it is supported [3]_.
+
+Now install thrift, for instructions see [4]_.
+If every thing went fine you are ready to generate the method stubs, the command basically looks like this. ::
+
+     $ thrift --gen (language) pyload.thrift
+
+You find now a directory named :file:`gen-(language)`. For instruction how to use the generated files consider the docs
+at the thrift wiki, as well at the examples [5]_.
+
+
+Example
+-------
+
+In case you want to use python, pyload has already all files included to access the api over rpc.
+
+A basic script that prints out some information: ::
+
+    from module.remote.thriftbackend.ThriftClient import ThriftClient, WrongLogin
+
+    try:
+        client = ThriftClient(host="127.0.0.1", port=7227, user="User", password="yourpw")
+    except:
+        print "Login was wrong"
+        exit()
+
+    print "Server version:", client.getServerVersion()
+    print client.statusDownloads()
+    q = client.getQueue()
+    for p in q:
+      data = client.getPackageData(p.pid)
+      print "Package Name: ", data.name
+
+That's all for now, pretty easy isn't it?
+If you still have open questions come around in irc or post them at our pyload forum.
+
+.. rubric:: Footnotes
+
+.. [1] http://en.wikipedia.org/wiki/Thrift_(protocol)
+.. [2] http://wiki.apache.org/thrift/
+.. [3] http://wiki.apache.org/thrift/LibraryFeatures?action=show&redirect=LanguageSupport
+.. [4] http://wiki.apache.org/thrift/ThriftInstallation
+.. [5] http://wiki.apache.org/thrift/ThriftUsage
\ No newline at end of file