diff options
193 files changed, 6630 insertions, 8513 deletions
@@ -29,7 +29,7 @@ Required - pycurl a.k.a python-curl - jinja2 - beaker -- thrift +- thrift >= 0.8 - simplejson (for python 2.5) Some plugins require additional packages, only install these when needed. diff --git a/docs/_static/logo.png b/docs/_static/logo.png Binary files differnew file mode 100644 index 000000000..1a11f5cc0 --- /dev/null +++ b/docs/_static/logo.png diff --git a/docs/access_api.rst b/docs/access_api.rst deleted file mode 100644 index df69da8b2..000000000 --- a/docs/access_api.rst +++ /dev/null @@ -1,121 +0,0 @@ -.. _access_api: - -********************* -How to access the API -********************* - -pyLoad has a very powerfull API with can be accessed in several ways. - -Overview --------- - -First of all, you need to know what you can do with our API. It lets you do all common task like -retrieving download status, manage queue, manage accounts, modify config and so on. - -This document is not intended to explain every function in detail, for a complete listing -see :class:`Api <module.Api.Api>`. - -Of course its possible to access the ``core.api`` attribute in plugins and hooks, but much more -interesting is the possibillity to call function from different programs written in many different languages. - -pyLoad uses thrift as backend and provides its :class:`Api <module.Api.Api>` as service. -More information about thrift can be found here http://wiki.apache.org/thrift/. - - -Using Thrift ------------- - -Every thrift service has to define all data structures and declare every method which should be usable via rpc. -This file is located :file:`module/remote/thriftbackend/pyload.thrift`, its very helpful to inform about -arguments and detailed structure of return types. However it does not contain any information about what the functions does. - -Assuming you want to use the API in any other language than python than check if it is -supported here http://wiki.apache.org/thrift/LibraryFeatures?action=show&redirect=LanguageSupport. - -Now install thrift, for instructions see http://wiki.apache.org/thrift/ThriftInstallation. -If every thing went fine you are ready to generate the method stubs, the command basically looks like this. :: - - $ thrift --gen (language) pyload.thrift - -You find now a directory named :file:`gen-(language)`. For instruction how to use the generated files consider the docs -at the thrift wiki and the examples here http://wiki.apache.org/thrift/ThriftUsage. - - -======= -Example -======= -In case you want to use python, pyload has already all files included to access the api over rpc. - -A basic script that prints out some information: :: - - from module.remote.thriftbackend.ThriftClient import ThriftClient, WrongLogin - - try: - client = ThriftClient(host="127.0.0.1", port=7227, user="User", password="yourpw") - except: - print "Login was wrong" - exit() - - print "Server version:", client.getServerVersion() - print client.statusDownloads() - q = client.getQueue() - for p in q: - data = client.getPackageData(p.pid) - print "Package Name: ", data.name - -That's all for now, pretty easy isn't it? -If you still have open questions come around in irc or post them at our pyload forum. - - -Using HTTP/JSON ---------------- - -Another maybe easier way, which does not require much setup is to access the JSON Api via HTTP. -For this reason the webinterface must be enabled. - -===== -Login -===== - -First you need to authenticate, if you using this within the webinterface and the user is logged the API is also accessible, -since they share the same cookie/session. - -However, if you are building a external client and want to authenticate manually -you have to send your credentials ``username`` and ``password`` as -POST parameter to ``http://pyload-core/api/login``. - -The result will be your session id. If you are using cookies, it will be set and you can use the API now. -In case you dont't have cookies enabled you can pass the session id as ``session`` POST parameter -so pyLoad can authenticate you. - -=============== -Calling Methods -=============== - -In general you can use any method listed at the :class:`Api <module.Api.Api>` documentation, which is also available to -the thriftbackend. - -Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address -or hostname including the webinterface port. By default on local access this would be `localhost:8000`. - -The return value will be formatted in JSON, complex data types as dictionaries. -As mentionted above for a documentation about the return types look at the thrift specification file :file:`module/remote/thriftbackend/pyload.thrift`. - -================== -Passing parameters -================== - -To pass arguments you have two choices. -Either use positional arguments, eg ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword arguments -supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names in the :class:`Api <module.Api.Api>` -documentation. - -It is important that *all* arguments are in JSON format. So ``http://pyload-core/api/getFileData/1`` is valid because -1 represents an integer in json format. On the other hand if the method is expecting strings, this would be correct: -``http://pyload-core/api/getUserData/"username"/"password"``. - -Strings are wrapped in double qoutes, because `"username"` represents a string in json format. It's not limited to strings and intergers, -every container type like lists and dicts are possible. You usually don't have to convert them. just use a json encoder before using them -in the HTTP request. - -Please note that the data have to be urlencoded at last. (Most libaries will do that automatically)
\ No newline at end of file diff --git a/docs/api/datatypes.rst b/docs/api/datatypes.rst new file mode 100644 index 000000000..02e3209cc --- /dev/null +++ b/docs/api/datatypes.rst @@ -0,0 +1,352 @@ +.. _api_datatypes: + +*********************** +API Datatype Definition +*********************** + +Below you find a copy of :file:`module/remote/thriftbackend/pyload.thrift`, which is used to generate the data structs +for various languages. It is also a good overview of avaible methods and return data. + +.. code-block:: c + + .. [[[cog cog.out(open('module/remote/thriftbackend/pyload.thrift', 'rb').read()) ]]] + namespace java org.pyload.thrift + + typedef i32 FileID + typedef i32 PackageID + typedef i32 ResultID + typedef i32 InteractionID + typedef list<string> LinkList + typedef string PluginName + typedef byte Progress + typedef byte Priority + + + enum DownloadStatus { + Finished + Offline, + Online, + Queued, + Skipped, + Waiting, + TempOffline, + Starting, + Failed, + Aborted, + Decrypting, + Custom, + Downloading, + Processing, + Unknown + } + + enum Destination { + Collector, + Queue + } + + // types for user interaction + // some may only be place holder currently not supported + // also all input - output combination are not reasonable, see InteractionManager for further info + enum Input { + NONE, + TEXT, + TEXTBOX, + PASSWORD, + BOOL, // confirm like, yes or no dialog + CLICK, // for positional captchas + CHOICE, // choice from list + MULTIPLE, // multiple choice from list of elements + LIST, // arbitary list of elements + TABLE // table like data structure + } + // more can be implemented by need + + // this describes the type of the outgoing interaction + // ensure they can be logcial or'ed + enum Output { + CAPTCHA = 1, + QUESTION = 2, + NOTIFICATION = 4, + } + + struct DownloadInfo { + 1: FileID fid, + 2: string name, + 3: i64 speed, + 4: i32 eta, + 5: string format_eta, + 6: i64 bleft, + 7: i64 size, + 8: string format_size, + 9: Progress percent, + 10: DownloadStatus status, + 11: string statusmsg, + 12: string format_wait, + 13: i64 wait_until, + 14: PackageID packageID, + 15: string packageName, + 16: PluginName plugin, + } + + struct ServerStatus { + 1: bool pause, + 2: i16 active, + 3: i16 queue, + 4: i16 total, + 5: i64 speed, + 6: bool download, + 7: bool reconnect + } + + struct FileData { + 1: FileID fid, + 2: string url, + 3: string name, + 4: PluginName plugin, + 5: i64 size, + 6: string format_size, + 7: DownloadStatus status, + 8: string statusmsg, + 9: PackageID packageID, + 10: string error, + 11: i16 order + } + + struct PackageData { + 1: PackageID pid, + 2: string name, + 3: string folder, + 4: string site, + 5: string password, + 6: Destination dest, + 7: i16 order, + 8: optional i16 linksdone, + 9: optional i64 sizedone, + 10: optional i64 sizetotal, + 11: optional i16 linkstotal, + 12: optional list<FileData> links, + 13: optional list<FileID> fids + } + + struct InteractionTask { + 1: InteractionID iid, + 2: Input input, + 3: list<string> structure, + 4: list<string> preset, + 5: Output output, + 6: list<string> data, + 7: string title, + 8: string description, + 9: string plugin, + } + + struct ConfigItem { + 1: string name, + 2: string long_name, + 3: string description, + 4: string type, + 5: string default_value, + 6: string value, + } + + struct ConfigSection { + 1: string name, + 2: string long_name, + 3: string description, + 4: string long_description, + 5: optional list<ConfigItem> items, + 6: optional map<string, InteractionTask> handler, + } + + struct CaptchaTask { + 1: i16 tid, + 2: binary data, + 3: string type, + 4: string resultType + } + + struct EventInfo { + 1: string eventname, + 2: list<string> event_args, + } + + struct UserData { + 1: string name, + 2: string email, + 3: i32 role, + 4: i32 permission, + 5: string templateName + } + + struct AccountInfo { + 1: PluginName plugin, + 2: string loginname, + 3: bool valid, + 4: i64 validuntil, + 5: i64 trafficleft, + 6: i64 maxtraffic, + 7: bool premium, + 8: bool activated, + 9: map<string, string> options, + } + + struct ServiceCall { + 1: PluginName plugin, + 2: string func, + 3: string arguments, // empty string or json encoded list + } + + struct OnlineStatus { + 1: string name, + 2: PluginName plugin, + 3: string packagename, + 4: DownloadStatus status, + 5: i64 size, // size <= 0 : unknown + } + + struct OnlineCheck { + 1: ResultID rid, // -1 -> nothing more to get + 2: map<string, OnlineStatus> data, //url to result + } + + + // exceptions + + exception PackageDoesNotExists{ + 1: PackageID pid + } + + exception FileDoesNotExists{ + 1: FileID fid + } + + exception UserDoesNotExists{ + 1: string user + } + + exception ServiceDoesNotExists{ + 1: string plugin + 2: string func + } + + exception ServiceException{ + 1: string msg + } + + service Pyload { + + //config + string getConfigValue(1: string section, 2: string option), + void setConfigValue(1: string section, 2: string option, 3: string value), + map<string, ConfigSection> getConfig(), + map<PluginName, ConfigSection> getPluginConfig(), + ConfigSection configureSection(1: string section), + + // server status + void pauseServer(), + void unpauseServer(), + bool togglePause(), + ServerStatus statusServer(), + i64 freeSpace(), + string getServerVersion(), + void kill(), + void restart(), + list<string> getLog(1: i32 offset), + bool isTimeDownload(), + bool isTimeReconnect(), + bool toggleReconnect(), + + // download preparing + + // packagename - urls + map<string, LinkList> generatePackages(1: LinkList links), + map<PluginName, LinkList> checkURLs(1: LinkList urls), + map<PluginName, LinkList> parseURLs(1: string html, 2: string url), + + // parses results and generates packages + OnlineCheck checkOnlineStatus(1: LinkList urls), + OnlineCheck checkOnlineStatusContainer(1: LinkList urls, 2: string filename, 3: binary data) + + // poll results from previosly started online check + OnlineCheck pollResults(1: ResultID rid), + + // downloads - information + list<DownloadInfo> statusDownloads(), + PackageData getPackageData(1: PackageID pid) throws (1: PackageDoesNotExists e), + PackageData getPackageInfo(1: PackageID pid) throws (1: PackageDoesNotExists e), + FileData getFileData(1: FileID fid) throws (1: FileDoesNotExists e), + list<PackageData> getQueue(), + list<PackageData> getCollector(), + list<PackageData> getQueueData(), + list<PackageData> getCollectorData(), + map<i16, PackageID> getPackageOrder(1: Destination destination), + map<i16, FileID> getFileOrder(1: PackageID pid) + + // downloads - adding/deleting + list<PackageID> generateAndAddPackages(1: LinkList links, 2: Destination dest), + PackageID addPackage(1: string name, 2: LinkList links, 3: Destination dest, 4: string password), + void addFiles(1: PackageID pid, 2: LinkList links), + void uploadContainer(1: string filename, 2: binary data), + void deleteFiles(1: list<FileID> fids), + void deletePackages(1: list<PackageID> pids), + + // downloads - modifying + void pushToQueue(1: PackageID pid), + void pullFromQueue(1: PackageID pid), + void restartPackage(1: PackageID pid), + void restartFile(1: FileID fid), + void recheckPackage(1: PackageID pid), + void stopAllDownloads(), + void stopDownloads(1: list<FileID> fids), + void setPackageName(1: PackageID pid, 2: string name), + void movePackage(1: Destination destination, 2: PackageID pid), + void moveFiles(1: list<FileID> fids, 2: PackageID pid), + void orderPackage(1: PackageID pid, 2: i16 position), + void orderFile(1: FileID fid, 2: i16 position), + void setPackageData(1: PackageID pid, 2: map<string, string> data) throws (1: PackageDoesNotExists e), + list<PackageID> deleteFinished(), + void restartFailed(), + + //events + list<EventInfo> getEvents(1: string uuid) + + //accounts + list<AccountInfo> getAccounts(1: bool refresh), + list<string> getAccountTypes() + void updateAccount(1: PluginName plugin, 2: string account, 3: string password, 4: map<string, string> options), + void removeAccount(1: PluginName plugin, 2: string account), + + //auth + bool login(1: string username, 2: string password), + UserData getUserData(1: string username, 2:string password) throws (1: UserDoesNotExists ex), + map<string, UserData> getAllUserData(), + + //services + + // servicename : description + map<PluginName, map<string, string>> getServices(), + bool hasService(1: PluginName plugin, 2: string func), + string call(1: ServiceCall info) throws (1: ServiceDoesNotExists ex, 2: ServiceException e), + + + //info + // {plugin: {name: value}} + map<PluginName, map<string,string>> getAllInfo(), + map<string, string> getInfoByPlugin(1: PluginName plugin), + + //scheduler + + // TODO + + + // User interaction + + //captcha + bool isCaptchaWaiting(), + CaptchaTask getCaptchaTask(1: bool exclusive), + string getCaptchaTaskStatus(1: InteractionID tid), + void setCaptchaResult(1: InteractionID tid, 2: string result), + } + .. [[[end]]] + diff --git a/docs/api/json_api.rst b/docs/api/json_api.rst new file mode 100644 index 000000000..3df006c49 --- /dev/null +++ b/docs/api/json_api.rst @@ -0,0 +1,72 @@ +.. _json_api: + +======== +JSON API +======== + +JSON [1]_ is a lightweight object notation and wrapper exists for nearly every programming language. Every +modern browser is able to load JSON objects with JavaScript. Unlike to thrift you don't need to generate or precompile +any stub methods, the JSON :class:`Api <module.Api.Api>` is ready to use for most language. The libary is really lightweight (at least in python) +and you can build very lightweight scripts with it. Because of the builtin support JSON is the first choice for all browser +applications. + +In our case JSON is just the output format, you have exactly the same methods available as with the thrift backend. The only +difference is the used protocol. + +So are there still reasons to choose the original :doc:`thrift <thrift_api>` backend in favor to JSON? Yes, since it +uses a binary protocol the performance will be better (when generating the objects), traffic will be smaller and +therefore the transfer faster. +In most IDEs you will get code completion, because of the pre-generated classes, which can make work much easier. + +If you intend to write a full client you should prefer thrift if the language is supported, for lightweight scripts and +in browser environment JSON wil be the better choice. + +Login +----- + +First you need to authenticate, if you using this within the webinterface and the user is logged the API is also accessible, +since they share the same cookie/session. + +However, if you are building a external client and want to authenticate manually +you have to send your credentials ``username`` and ``password`` as +POST parameter to ``http://pyload-core/api/login``. + +The result will be your session id. If you are using cookies, it will be set and you can use the API now. +In case you dont't have cookies enabled you can pass the session id as ``session`` POST parameter +so pyLoad can authenticate you. + + +Calling Methods +--------------- + +In general you can use any method listed at the :class:`Api <module.Api.Api>` documentation, which is also available to +the thriftbackend. + +Access works simply via ``http://pyload-core/api/methodName``, where ``pyload-core`` is the ip address +or hostname including the webinterface port. By default on local access this would be `localhost:8000`. + +The return value will be formatted in JSON, complex data types as dictionaries. Definition for datatypes can be found +:doc:`here <datatypes>` + +Passing parameters +------------------ + +To pass arguments you have two choices. +Either use positional arguments, eg ``http://pyload-core/api/getFileData/1``, where 1 is the FileID, or use keyword +arguments supplied via GET or POST ``http://pyload-core/api/getFileData?fid=1``. You can find the argument names +in the :class:`Api <module.Api.Api>` documentation. + +It is important that *all* arguments are in JSON format. So ``http://pyload-core/api/getFileData/1`` is valid because +1 represents an integer in json format. On the other hand if the method is expecting strings, this would be correct: +``http://pyload-core/api/getUserData/"username"/"password"``. + +Strings are wrapped in double qoutes, because `"username"` represents a string in json format. It's not limited to +strings and intergers, every container type like lists and dicts are possible. You usually don't have to convert them. +Just use a json encoder before using them in the HTTP request. + +Please note that the data have to be urlencoded at last. (Most libaries will do that automatically) + + +.. rubric:: Footnotes + +.. [1] http://de.wikipedia.org/wiki/JavaScript_Object_Notation
\ No newline at end of file diff --git a/docs/api/overview.rst b/docs/api/overview.rst new file mode 100644 index 000000000..47fe1be82 --- /dev/null +++ b/docs/api/overview.rst @@ -0,0 +1,36 @@ +.. _overview: + +======================================= +API - Application Programming Interface +======================================= + +From Wikipedia, the free encyclopedia [1]_: + + An application programming interface (API) is a source code based specification intended to be used as an interface + by software components to communicate with each other. An API may include specifications for routines, + data structures, object classes, and variables. + +.. rubric:: Motivation + +The idea of the centralized pyLoad :class:`Api <module.Api.Api>` is to give uniform access to all integral parts +and plugins in pyLoad, and furthermore to other clients, written in arbitrary programming languages. +Most of the :class:`Api <module.Api.Api>` functionality is exposed via RPC [2]_ and accessible via thrift [3]_ or +simple JSON objects [4]_. In conclusion the :class:`Api <module.Api.Api>` is accessible via many programming language, +over network from remote maschines and over browser with javascript. + + +.. rubric:: Contents + +.. toctree:: + + thrift_api.rst + json_api.rst + datatypes.rst + + +.. rubric:: Footnotes + +.. [1] http://en.wikipedia.org/wiki/Application_programming_interface +.. [2] http://en.wikipedia.org/wiki/Remote_procedure_call +.. [3] `<http://en.wikipedia.org/wiki/Thrift_(protocol)>`_ +.. [4] http://en.wikipedia.org/wiki/Json
\ No newline at end of file diff --git a/docs/api/thrift_api.rst b/docs/api/thrift_api.rst new file mode 100644 index 000000000..a4987a797 --- /dev/null +++ b/docs/api/thrift_api.rst @@ -0,0 +1,74 @@ +.. _thrift_api: + +========== +Thrift API +========== + +Thrift [1]_ was first developed in-house at facebook, but later published to public domain and developed at Apache Incubator. +It includes a binary protocol for remote calls, which is much more performant than other data formats like XML, additionally +it is available for numerous languages and therefore we choosed it as primary backend for our API. + +First of all, you need to know what you can do with our API. It lets you do all common task like +retrieving download status, manage queue, manage accounts, modify config and so on. + +This document is not intended to explain every function in detail, for a complete listing +see :class:`Api <module.Api.Api>`. + +Of course its possible to access the ``core.api`` attribute in plugins and hooks, but much more +interesting is the possibillity to call function from different programs written in many different languages. + +pyLoad uses thrift as backend and provides its :class:`Api <module.Api.Api>` as service. +More information about thrift can be found in their wiki [2]_. + + +Using Thrift +------------ + +Every thrift service has to define all data structures and declare every method which should be usable via rpc. +This file is located at :file:`module/remote/thriftbackend/pyload.thrift`, its very helpful to inform about +arguments and detailed structure of return types. However it does not contain any information about what the functions does. +You can also look at it :doc:`here <datatypes>` + +Assuming you want to use the API in any other language than python than check if it is supported [3]_. + +Now install thrift, for instructions see [4]_. +If every thing went fine you are ready to generate the method stubs, the command basically looks like this. :: + + $ thrift --gen (language) pyload.thrift + +You find now a directory named :file:`gen-(language)`. For instruction how to use the generated files consider the docs +at the thrift wiki, as well at the examples [5]_. + + +Example +------- + +In case you want to use python, pyload has already all files included to access the api over rpc. + +A basic script that prints out some information: :: + + from module.remote.thriftbackend.ThriftClient import ThriftClient, WrongLogin + + try: + client = ThriftClient(host="127.0.0.1", port=7227, user="User", password="yourpw") + except: + print "Login was wrong" + exit() + + print "Server version:", client.getServerVersion() + print client.statusDownloads() + q = client.getQueue() + for p in q: + data = client.getPackageData(p.pid) + print "Package Name: ", data.name + +That's all for now, pretty easy isn't it? +If you still have open questions come around in irc or post them at our pyload forum. + +.. rubric:: Footnotes + +.. [1] http://en.wikipedia.org/wiki/Thrift_(protocol) +.. [2] http://wiki.apache.org/thrift/ +.. [3] http://wiki.apache.org/thrift/LibraryFeatures?action=show&redirect=LanguageSupport +.. [4] http://wiki.apache.org/thrift/ThriftInstallation +.. [5] http://wiki.apache.org/thrift/ThriftUsage
\ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 9d2cf98f9..4961fc910 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,11 +27,12 @@ sys.path.append(join(dir_name, "module", "lib")) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', + 'sphinx.ext.pngmath', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] autosummary_generate = True autodoc_default_flags = ['members'] @@ -52,7 +53,7 @@ master_doc = 'index' # General information about the project. project = u'pyLoad' -copyright = u'2011, pyLoad Team' +copyright = u'2012, pyLoad Team' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -65,8 +66,8 @@ v = options.version.split(".") cog.outl("version = '%s'" % ".".join(v[:2])) cog.outl("release = '%s'" % ".".join(v)) ]]]""" -version = '0.4' -release = '0.4.9' +version = '0.5' +release = '0.5.0' # [[[end]]] @@ -196,8 +197,8 @@ htmlhelp_basename = 'pyLoaddoc' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'pyLoad.tex', u'pyLoad Documentation', - u'pyLoad Team', 'manual'), + ('index', 'pyLoad.tex', u'pyLoad Documentation', + u'pyLoad Team', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -235,4 +236,4 @@ man_pages = [ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = {'http://docs.python.org/': None}
\ No newline at end of file diff --git a/docs/extend_pyload.rst b/docs/extend_pyload.rst deleted file mode 100755 index 337cb6854..000000000 --- a/docs/extend_pyload.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _extend_pyload: - -******************** -How to extend pyLoad -******************** - -In general there a two different plugin types. These allow everybody to write powerful, modular plugins without knowing -every detail of the pyLoad core. However you should have some basic knowledge of python. - -.. toctree:: - - write_hooks.rst - write_plugins.rst
\ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 757fd7537..31d688e65 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,27 +1,47 @@ -.. pyLoad documentation master file, created by - sphinx-quickstart on Sat Jun 4 11:54:34 2011. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. pyLoad documentation master file -Welcome to pyLoad's documentation! -================================== +===================== +pyLoad Documentation +===================== -Great that you found your way to the pyLoad documentation! +.. image:: _static/logo.png + :height: 144 + :width: 412 -We have collected some information here to help developer writing plugins and understandig our code. -If you want to help us developing visit us in our IRC channel #pyload on freenode.net or leave a message in our forum. -Contents: +Great that you found your way to the pyLoad [1]_ documentation! + +This is the ultimate document to get started extending or accessing pyLoad in your own way. +We will cover on how to access the API so you can write your own client to pyLoad. In the next step you will be given +an idea on how to extend pyLoad and write your own powerful plugins, which perfectly integrate into our system. + +The complete pyLoad source and this documentation is available at bitbucket [2]_. If you would like to contribute +come around in our irc channel [3]_ or open a pull request. +In case you still have questions, ask at our forum [4]_ or in our official irc channel #pyload @ irc.freenode.net + +We wish you happy programming! + +-- the pyLoad Team + +Contents +-------- .. toctree:: :maxdepth: 2 - access_api.rst - extend_pyload.rst + api/overview.rst + plugins/overview.rst module_overview.rst .. currentmodule:: module +.. rubric:: Footnotes + +.. [1] http://pyload.org +.. [2] http://pyload.org/irc +.. [3] http://bitbucket.org/spoob/pyload/overview +.. [4] http://forum.pyload.org + ================== * :ref:`genindex` diff --git a/docs/module_overview.rst b/docs/module_overview.rst index d51202c88..b8db51538 100644 --- a/docs/module_overview.rst +++ b/docs/module_overview.rst @@ -1,17 +1,22 @@ + Module Overview =============== -You can find an overview of some important classes here: +A little selection of most important modules in pyLoad. .. autosummary:: :toctree: module module.Api.Api - module.plugins.Plugin.Base - module.plugins.Plugin.Plugin + module.plugins.Base.Base + module.plugins.Hoster.Hoster module.plugins.Crypter.Crypter - module.plugins.Account.Account module.plugins.Hook.Hook + module.plugins.Account.Account + module.plugins.MultiHoster.MultiHoster module.HookManager.HookManager + module.interaction.EventManager.EventManager + module.interaction.InteractionManager.InteractionManager + module.interaction.InteractionTask.InteractionTask module.PyFile.PyFile module.PyPackage.PyPackage diff --git a/docs/plugins/account_plugin.rst b/docs/plugins/account_plugin.rst new file mode 100644 index 000000000..75bf61a75 --- /dev/null +++ b/docs/plugins/account_plugin.rst @@ -0,0 +1,5 @@ +.. _account_plugin: + +Account - Premium Access +======================== + diff --git a/docs/plugins/base_plugin.rst b/docs/plugins/base_plugin.rst new file mode 100644 index 000000000..4ffe2e457 --- /dev/null +++ b/docs/plugins/base_plugin.rst @@ -0,0 +1,24 @@ +.. _base_plugin: + +Base Plugin - And here it begins... +=================================== + + +Meta Data +--------- + + +Config Entries +-------------- + + +Tagging Guidelines +------------------ + + +Basic Methods +------------- + +Debugging +--------- + diff --git a/docs/plugins/crypter_plugin.rst b/docs/plugins/crypter_plugin.rst new file mode 100644 index 000000000..d910ec412 --- /dev/null +++ b/docs/plugins/crypter_plugin.rst @@ -0,0 +1,5 @@ +.. _crypter_plugin: + +Crypter - Extract links from pages +================================== + diff --git a/docs/write_hooks.rst b/docs/plugins/hook_plugin.rst index dd60367b7..be1097057 100644 --- a/docs/write_hooks.rst +++ b/docs/plugins/hook_plugin.rst @@ -1,7 +1,7 @@ .. _write_hooks: -Hooks -===== +Hook - Do whatever you want +============================= A Hook is a python file which is located at :file:`module/plugins/hooks`. The :class:`HookManager <module.HookManager.HookManager>` will load it automatically on startup. Only one instance exists diff --git a/docs/write_plugins.rst b/docs/plugins/hoster_plugin.rst index b513a5978..59f35f5cb 100644 --- a/docs/write_plugins.rst +++ b/docs/plugins/hoster_plugin.rst @@ -1,7 +1,7 @@ -.. _write_plugins: +.. _hoster_plugin: -Plugins -======= +Hoster - Load files to disk +=========================== A Plugin is a python file located at one of the subfolders in :file:`module/plugins/`. Either :file:`hoster`, :file:`crypter` or :file:`container`, depending of it's type. @@ -22,7 +22,6 @@ How basic hoster plugin header could look like: :: from module.plugin.Hoster import Hoster class MyFileHoster(Hoster): - __name__ = "MyFileHoster" __version__ = "0.1" __pattern__ = r"http://myfilehoster.example.com/file_id/[0-9]+" __config__ = [] diff --git a/docs/plugins/overview.rst b/docs/plugins/overview.rst new file mode 100755 index 000000000..23913b787 --- /dev/null +++ b/docs/plugins/overview.rst @@ -0,0 +1,31 @@ +.. _overview: + +================ +Extending pyLoad +================ + +.. pull-quote:: + Any sufficiently advanced technology is indistinguishable from magic. + + -- Arthur C. Clarke + + +.. rubric:: Motivation + +pyLoad offers an comfortable and powerful plugin system to make extending possible. With it you only need to have some +python knowledge and can just start right away writing your own plugins. This document gives you an overwiew about the +conceptual part. You should not left out the `Base` part, since it contains basic functionality for all plugins types. + +.. rubric:: Contents + +.. toctree:: + + base_plugin.rst + crypter_plugin.rst + hoster_plugin.rst + account_plugin.rst + hook_plugin.rst + + + +.. rubric:: Footnotes
\ No newline at end of file diff --git a/icons/abort.png b/icons/abort.png Binary files differdeleted file mode 100644 index 66170aae7..000000000 --- a/icons/abort.png +++ /dev/null diff --git a/icons/add_small.png b/icons/add_small.png Binary files differdeleted file mode 100644 index 4dc88b09b..000000000 --- a/icons/add_small.png +++ /dev/null diff --git a/icons/clipboard.png b/icons/clipboard.png Binary files differdeleted file mode 100644 index 9ba608eba..000000000 --- a/icons/clipboard.png +++ /dev/null diff --git a/icons/close.png b/icons/close.png Binary files differdeleted file mode 100644 index 66170aae7..000000000 --- a/icons/close.png +++ /dev/null diff --git a/icons/edit_small.png b/icons/edit_small.png Binary files differdeleted file mode 100644 index eb76e21b4..000000000 --- a/icons/edit_small.png +++ /dev/null diff --git a/icons/logo-gui.png b/icons/logo-gui.png Binary files differdeleted file mode 100644 index 5994b274d..000000000 --- a/icons/logo-gui.png +++ /dev/null diff --git a/icons/logo.png b/icons/logo.png Binary files differdeleted file mode 100644 index 72a95b740..000000000 --- a/icons/logo.png +++ /dev/null diff --git a/icons/pull_small.png b/icons/pull_small.png Binary files differdeleted file mode 100644 index 432ad321f..000000000 --- a/icons/pull_small.png +++ /dev/null diff --git a/icons/push_small.png b/icons/push_small.png Binary files differdeleted file mode 100644 index 701fc69e3..000000000 --- a/icons/push_small.png +++ /dev/null diff --git a/icons/pyload-gui.ico b/icons/pyload-gui.ico Binary files differdeleted file mode 100644 index 00a1a53ff..000000000 --- a/icons/pyload-gui.ico +++ /dev/null diff --git a/icons/pyload.ico b/icons/pyload.ico Binary files differdeleted file mode 100644 index 58b1f4b89..000000000 --- a/icons/pyload.ico +++ /dev/null diff --git a/icons/pyload2.ico b/icons/pyload2.ico Binary files differdeleted file mode 100644 index c2b497986..000000000 --- a/icons/pyload2.ico +++ /dev/null diff --git a/icons/refresh1_small.png b/icons/refresh1_small.png Binary files differdeleted file mode 100644 index ce4f24efc..000000000 --- a/icons/refresh1_small.png +++ /dev/null diff --git a/icons/refresh_small.png b/icons/refresh_small.png Binary files differdeleted file mode 100644 index 1ffd18d97..000000000 --- a/icons/refresh_small.png +++ /dev/null diff --git a/icons/remove_small.png b/icons/remove_small.png Binary files differdeleted file mode 100644 index bf99763e8..000000000 --- a/icons/remove_small.png +++ /dev/null diff --git a/icons/toolbar_add.png b/icons/toolbar_add.png Binary files differdeleted file mode 100644 index 17003e9f0..000000000 --- a/icons/toolbar_add.png +++ /dev/null diff --git a/icons/toolbar_pause.png b/icons/toolbar_pause.png Binary files differdeleted file mode 100644 index b7a727b71..000000000 --- a/icons/toolbar_pause.png +++ /dev/null diff --git a/icons/toolbar_remove.png b/icons/toolbar_remove.png Binary files differdeleted file mode 100644 index 1e9c00e16..000000000 --- a/icons/toolbar_remove.png +++ /dev/null diff --git a/icons/toolbar_start.png b/icons/toolbar_start.png Binary files differdeleted file mode 100644 index 1123266e6..000000000 --- a/icons/toolbar_start.png +++ /dev/null diff --git a/icons/toolbar_stop.png b/icons/toolbar_stop.png Binary files differdeleted file mode 100644 index b388e3d72..000000000 --- a/icons/toolbar_stop.png +++ /dev/null diff --git a/module/Api.py b/module/Api.py index f0bf5e264..e5d26631f 100644 --- a/module/Api.py +++ b/module/Api.py @@ -17,13 +17,16 @@ @author: RaNaN """ +import re from base64 import standard_b64encode -from os.path import join +from os.path import join, isabs from time import time -import re +from itertools import chain + from PyFile import PyFile -from utils import freeSpace, compare_time +from utils import compare_time, to_string +from utils.fs import free_space from common.packagetools import parseNames from network.RequestFactory import getURL from remote import activated @@ -32,6 +35,7 @@ if activated: try: from remote.thriftbackend.thriftgen.pyload.ttypes import * from remote.thriftbackend.thriftgen.pyload.Pyload import Iface + BaseObject = TBase except ImportError: print "Thrift not imported" @@ -49,7 +53,7 @@ def permission(bits): def __new__(cls, func, *args, **kwargs): permMap[func.__name__] = bits return func - + return _Dec @@ -67,10 +71,12 @@ class PERMS: ACCOUNTS = 256 # can access accounts LOGS = 512 # can see server logs + class ROLE: ADMIN = 0 #admin has all permissions implicit USER = 1 + def has_permission(userperms, perms): # bytewise or perms before if needed return perms == (userperms & perms) @@ -97,80 +103,45 @@ class Api(Iface): def _convertPyFile(self, p): f = FileData(p["id"], p["url"], p["name"], p["plugin"], p["size"], - p["format_size"], p["status"], p["statusmsg"], - p["package"], p["error"], p["order"]) + p["format_size"], p["status"], p["statusmsg"], + p["package"], p["error"], p["order"]) return f - def _convertConfigFormat(self, c): - sections = {} - for sectionName, sub in c.iteritems(): - section = ConfigSection(sectionName, sub["desc"]) - items = [] - for key, data in sub.iteritems(): - if key in ("desc", "outline"): - continue - item = ConfigItem() - item.name = key - item.description = data["desc"] - item.value = str(data["value"]) if not isinstance(data["value"], basestring) else data["value"] - item.type = data["type"] - items.append(item) - section.items = items - sections[sectionName] = section - if "outline" in sub: - section.outline = sub["outline"] - return sections - @permission(PERMS.SETTINGS) - def getConfigValue(self, category, option, section="core"): + def getConfigValue(self, section, option): """Retrieve config value. - :param category: name of category, or plugin + :param section: name of category, or plugin :param option: config option - :param section: 'plugin' or 'core' :return: config value as string """ - if section == "core": - value = self.core.config[category][option] - else: - value = self.core.config.getPlugin(category, option) - - return str(value) if not isinstance(value, basestring) else value + value = self.core.config.get(section, option) + return to_string(value) @permission(PERMS.SETTINGS) - def setConfigValue(self, category, option, value, section="core"): + def setConfigValue(self, section, option, value): """Set new config value. - :param category: + :param section: :param option: :param value: new config value - :param section: 'plugin' or 'core """ - self.core.hookManager.dispatchEvent("configChanged", category, option, value, section) - - if section == "core": - self.core.config[category][option] = value - - if option in ("limit_speed", "max_speed"): #not so nice to update the limit - self.core.requestFactory.updateBucket() + if option in ("limit_speed", "max_speed"): #not so nice to update the limit + self.core.requestFactory.updateBucket() - elif section == "plugin": - self.core.config.setPlugin(category, option, value) + self.core.config.set(section, option, value) @permission(PERMS.SETTINGS) def getConfig(self): """Retrieves complete config of core. - + :return: list of `ConfigSection` """ - return self._convertConfigFormat(self.core.config.config) - - def getConfigDict(self): - """Retrieves complete config in dict format, not for RPC. + return dict([(section, ConfigSection(section, data.name, data.description, data.long_desc, [ + ConfigItem(option, d.name, d.description, d.type, to_string(d.default), to_string(self.core.config.get(section, option))) for + option, d in data.config.iteritems()])) for + section, data in self.core.config.getBaseSections()]) - :return: dict - """ - return self.core.config.config @permission(PERMS.SETTINGS) def getPluginConfig(self): @@ -178,15 +149,25 @@ class Api(Iface): :return: list of `ConfigSection` """ - return self._convertConfigFormat(self.core.config.plugin) + return dict([(section, ConfigSection(section, + data.name, data.description, data.long_desc)) for + section, data in self.core.config.getPluginSections()]) - def getPluginConfigDict(self): - """Plugin config as dict, not for RPC. + def configureSection(self, section): + data = self.core.config.config[section] + sec = ConfigSection(section, data.name, data.description, data.long_desc) + sec.items = [ConfigItem(option, d.name, d.description, + d.type, to_string(d.default), to_string(self.core.config.get(section, option))) + for + option, d in data.config.iteritems()] - :return: dict - """ - return self.core.config.plugin + #TODO: config handler + + return sec + def getConfigPointer(self): + """Config instance, not for RPC""" + return self.core.config @permission(PERMS.STATUS) def pauseServer(self): @@ -219,13 +200,13 @@ class Api(Iface): @permission(PERMS.LIST) def statusServer(self): """Some general information about the current status of pyLoad. - + :return: `ServerStatus` """ serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), - self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, - not self.core.threadManager.pause and self.isTimeDownload(), - self.core.config['reconnect']['activated'] and self.isTimeReconnect()) + self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, + not self.core.threadManager.pause and self.isTimeDownload(), + self.core.config['reconnect']['activated'] and self.isTimeReconnect()) for pyfile in [x.active for x in self.core.threadManager.threads if x.active and isinstance(x.active, PyFile)]: serverStatus.speed += pyfile.getSpeed() #bytes/s @@ -235,7 +216,7 @@ class Api(Iface): @permission(PERMS.STATUS) def freeSpace(self): """Available free space at download directory in bytes""" - return freeSpace(self.core.config["general"]["download_folder"]) + return free_space(self.core.config["general"]["download_folder"]) @permission(PERMS.ALL) def getServerVersion(self): @@ -308,12 +289,13 @@ class Api(Iface): return data @permission(PERMS.ADD) - def addPackage(self, name, links, dest=Destination.Queue): + def addPackage(self, name, links, dest=Destination.Queue, password=""): """Adds a package, with links to desired destination. :param name: name of the new package :param links: list of urls :param dest: `Destination` + :param password: password as string, can be empty :return: package id of the new package """ if self.core.config['general']['folder_per_package']: @@ -321,19 +303,36 @@ class Api(Iface): else: folder = "" - folder = folder.replace("http://", "").replace(":", "").replace("/", "_").replace("\\", "_") - - pid = self.core.files.addPackage(name, folder, dest) + if isabs(folder): + folder = folder.replace("/", "_") - self.core.files.addLinks(links, pid) + folder = folder.replace("http://", "").replace(":", "").replace("\\", "_").replace("..", "") self.core.log.info(_("Added package %(name)s containing %(count)d links") % {"name": name, "count": len(links)}) - - self.core.files.save() + pid = self.core.files.addPackage(name, folder, dest, password) + self.addFiles(pid, links) return pid @permission(PERMS.ADD) + def addFiles(self, pid, links): + """Adds files to specific package. + + :param pid: package id + :param links: list of urls + """ + hoster, crypter = self.core.pluginManager.parseUrls(links) + + if hoster: + self.core.files.addLinks(hoster, pid) + + self.core.threadManager.createInfoThread(hoster, pid) + self.core.threadManager.createDecryptThread(crypter, pid) + + self.core.log.debug("Added %d links to package #%d " % (len(hoster), pid)) + self.core.files.save() + + @permission(PERMS.ADD) def parseURLs(self, html=None, url=None): """Parses html content or any arbitaty text for links and returns result of `checkURLs` @@ -360,10 +359,10 @@ class Api(Iface): :param urls: :return: {plugin: urls} """ - data = self.core.pluginManager.parseUrls(urls) + data, crypter = self.core.pluginManager.parseUrls(urls) plugins = {} - for url, plugin in data: + for url, plugin in chain(data, crypter): if plugin in plugins: plugins[plugin].append(url) else: @@ -373,15 +372,14 @@ class Api(Iface): @permission(PERMS.ADD) def checkOnlineStatus(self, urls): - """ initiates online status check + """ initiates online status check, will also decrypt files. :param urls: :return: initial set of data as `OnlineCheck` instance containing the result id """ - data = self.core.pluginManager.parseUrls(urls) - - rid = self.core.threadManager.createResultThread(data, False) + data, crypter = self.core.pluginManager.parseUrls(urls) + # initial result does not contain the crypter links tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data] data = parseNames(tmp) result = {} @@ -391,6 +389,9 @@ class Api(Iface): status.packagename = k result[url] = status + data.update(crypter) # hoster and crypter will be processed + rid = self.core.threadManager.createResultThread(data, False) + return OnlineCheck(rid, result) @permission(PERMS.ADD) @@ -405,8 +406,8 @@ class Api(Iface): th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb") th.write(str(data)) th.close() - - return self.checkOnlineStatus(urls + [th.name]) + urls.append(th.name) + return self.checkOnlineStatus(urls) @permission(PERMS.ADD) def pollResults(self, rid): @@ -445,18 +446,6 @@ class Api(Iface): return [self.addPackage(name, urls, dest) for name, urls in self.generatePackages(links).iteritems()] - @permission(PERMS.ADD) - def checkAndAddPackages(self, links, dest=Destination.Queue): - """Checks online status, retrieves names, and will add packages.\ - Because of this packages are not added immediatly, only for internal use. - - :param links: list of urls - :param dest: `Destination` - :return: None - """ - data = self.core.pluginManager.parseUrls(links) - self.core.threadManager.createResultThread(data, True) - @permission(PERMS.LIST) def getPackageData(self, pid): @@ -471,8 +460,8 @@ class Api(Iface): raise PackageDoesNotExists(pid) pdata = PackageData(data["id"], data["name"], data["folder"], data["site"], data["password"], - data["queue"], data["order"], - links=[self._convertPyFile(x) for x in data["links"].itervalues()]) + data["queue"], data["order"], + links=[self._convertPyFile(x) for x in data["links"].itervalues()]) return pdata @@ -484,13 +473,13 @@ class Api(Iface): :return: `PackageData` with .fid attribute """ data = self.core.files.getPackageData(int(pid)) - + if not data: raise PackageDoesNotExists(pid) pdata = PackageData(data["id"], data["name"], data["folder"], data["site"], data["password"], - data["queue"], data["order"], - fids=[int(x) for x in data["links"]]) + data["queue"], data["order"], + fids=[int(x) for x in data["links"]]) return pdata @@ -511,7 +500,7 @@ class Api(Iface): @permission(PERMS.DELETE) def deleteFiles(self, fids): """Deletes several file entries from pyload. - + :param fids: list of file ids """ for id in fids: @@ -538,9 +527,9 @@ class Api(Iface): :return: list of `PackageInfo` """ return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], - pack["password"], pack["queue"], pack["order"], - pack["linksdone"], pack["sizedone"], pack["sizetotal"], - pack["linkstotal"]) + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + pack["linkstotal"]) for pack in self.core.files.getInfoData(Destination.Queue).itervalues()] @permission(PERMS.LIST) @@ -551,9 +540,9 @@ class Api(Iface): :return: list of `PackageData` """ return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], - pack["password"], pack["queue"], pack["order"], - pack["linksdone"], pack["sizedone"], pack["sizetotal"], - links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) for pack in self.core.files.getCompleteData(Destination.Queue).itervalues()] @permission(PERMS.LIST) @@ -563,9 +552,9 @@ class Api(Iface): :return: list of `PackageInfo` """ return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], - pack["password"], pack["queue"], pack["order"], - pack["linksdone"], pack["sizedone"], pack["sizetotal"], - pack["linkstotal"]) + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + pack["linkstotal"]) for pack in self.core.files.getInfoData(Destination.Collector).itervalues()] @permission(PERMS.LIST) @@ -575,24 +564,11 @@ class Api(Iface): :return: list of `PackageInfo` """ return [PackageData(pack["id"], pack["name"], pack["folder"], pack["site"], - pack["password"], pack["queue"], pack["order"], - pack["linksdone"], pack["sizedone"], pack["sizetotal"], - links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) + pack["password"], pack["queue"], pack["order"], + pack["linksdone"], pack["sizedone"], pack["sizetotal"], + links=[self._convertPyFile(x) for x in pack["links"].itervalues()]) for pack in self.core.files.getCompleteData(Destination.Collector).itervalues()] - - @permission(PERMS.ADD) - def addFiles(self, pid, links): - """Adds files to specific package. - - :param pid: package id - :param links: list of urls - """ - self.core.files.addLinks(links, int(pid)) - - self.core.log.info(_("Added %(count)d links to package #%(package)d ") % {"count": len(links), "package": pid}) - self.core.files.save() - @permission(PERMS.MODIFY) def pushToQueue(self, pid): """Moves package from Collector to Queue. @@ -699,7 +675,7 @@ class Api(Iface): th.write(str(data)) th.close() - self.addPackage(th.name, [th.name], Destination.Queue) + return self.addPackage(th.name, [th.name]) @permission(PERMS.MODIFY) def orderPackage(self, pid, position): @@ -839,31 +815,11 @@ class Api(Iface): def getEvents(self, uuid): """Lists occured events, may be affected to changes in future. - :param uuid: + :param uuid: self assigned string uuid which has to be unique :return: list of `Events` """ - events = self.core.pullManager.getEvents(uuid) - newEvents = [] - - def convDest(d): - return Destination.Queue if d == "queue" else Destination.Collector - - for e in events: - event = EventInfo() - event.eventname = e[0] - if e[0] in ("update", "remove", "insert"): - event.id = e[3] - event.type = ElementType.Package if e[2] == "pack" else ElementType.File - event.destination = convDest(e[1]) - elif e[0] == "order": - if e[1]: - event.id = e[1] - event.type = ElementType.Package if e[2] == "pack" else ElementType.File - event.destination = convDest(e[3]) - elif e[0] == "reload": - event.destination = convDest(e[1]) - newEvents.append(event) - return newEvents + # TODO + pass @permission(PERMS.ACCOUNTS) def getAccounts(self, refresh): @@ -872,21 +828,20 @@ class Api(Iface): :param refresh: reload account info :return: list of `AccountInfo` """ - accs = self.core.accountManager.getAccountInfos(False, refresh) + accs = self.core.accountManager.getAllAccounts(refresh) accounts = [] - for group in accs.values(): - accounts.extend([AccountInfo(acc["validuntil"], acc["login"], acc["options"], acc["valid"], - acc["trafficleft"], acc["maxtraffic"], acc["premium"], acc["type"]) - for acc in group]) + for plugin in accs.itervalues(): + accounts.extend(plugin.values()) + return accounts @permission(PERMS.ALL) def getAccountTypes(self): """All available account types. - :return: list + :return: string list """ - return self.core.accountManager.accounts.keys() + return self.core.pluginManager.getPlugins("accounts").keys() @permission(PERMS.ACCOUNTS) def updateAccount(self, plugin, account, password=None, options={}): @@ -946,11 +901,11 @@ class Api(Iface): @permission(PERMS.ALL) def getUserData(self, username, password): """similar to `checkAuth` but returns UserData thrift type """ - user = self.checkAuth(username, password) + user = self.checkAuth(username, password) if user: return UserData(user["name"], user["email"], user["role"], user["permission"], user["template"]) - else: - return UserData() + + raise UserDoesNotExists(username) def getAllUserData(self): @@ -996,13 +951,12 @@ class Api(Iface): plugin = info.plugin func = info.func args = info.arguments - parse = info.parseArguments if not self.hasService(plugin, func): raise ServiceDoesNotExists(plugin, func) try: - ret = self.core.hookManager.callRPC(plugin, func, args, parse) + ret = self.core.hookManager.callRPC(plugin, func, args) return str(ret) except Exception, e: raise ServiceException(e.message) @@ -1030,4 +984,4 @@ class Api(Iface): def setUserPermission(self, user, permission, role): self.core.db.setPermission(user, permission) - self.core.db.setRole(user, role)
\ No newline at end of file + self.core.db.setRole(user, role) diff --git a/module/ConfigParser.py b/module/ConfigParser.py deleted file mode 100644 index 78b612f13..000000000 --- a/module/ConfigParser.py +++ /dev/null @@ -1,392 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import with_statement -from time import sleep -from os.path import exists, join -from shutil import copy - -from traceback import print_exc -from utils import chmod - -# ignore these plugin configs, mainly because plugins were wiped out -IGNORE = ( - "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('hooks', 'UnRar'), - 'EasyShareCom', 'FlyshareCz' - ) - -CONF_VERSION = 1 - -class ConfigParser: - """ - holds and manage the configuration - - current dict layout: - - { - - section : { - option : { - value: - type: - desc: - } - desc: - - } - - - """ - - - def __init__(self): - """Constructor""" - self.config = {} # the config values - self.plugin = {} # the config for plugins - self.oldRemoteData = {} - - self.pluginCB = None # callback when plugin config value is changed - - self.checkVersion() - - self.readConfig() - - self.deleteOldPlugins() - - - def checkVersion(self, n=0): - """determines if config need to be copied""" - try: - if not exists("pyload.conf"): - copy(join(pypath, "module", "config", "default.conf"), "pyload.conf") - - if not exists("plugin.conf"): - f = open("plugin.conf", "wb") - f.write("version: " + str(CONF_VERSION)) - f.close() - - f = open("pyload.conf", "rb") - v = f.readline() - f.close() - v = v[v.find(":") + 1:].strip() - - if not v or int(v) < CONF_VERSION: - copy(join(pypath, "module", "config", "default.conf"), "pyload.conf") - print "Old version of config was replaced" - - f = open("plugin.conf", "rb") - v = f.readline() - f.close() - v = v[v.find(":") + 1:].strip() - - if not v or int(v) < CONF_VERSION: - f = open("plugin.conf", "wb") - f.write("version: " + str(CONF_VERSION)) - f.close() - print "Old version of plugin-config replaced" - except: - if n < 3: - sleep(0.3) - self.checkVersion(n + 1) - else: - raise - - def readConfig(self): - """reads the config file""" - - self.config = self.parseConfig(join(pypath, "module", "config", "default.conf")) - self.plugin = self.parseConfig("plugin.conf") - - try: - homeconf = self.parseConfig("pyload.conf") - if "username" in homeconf["remote"]: - if "password" in homeconf["remote"]: - self.oldRemoteData = {"username": homeconf["remote"]["username"]["value"], - "password": homeconf["remote"]["username"]["value"]} - del homeconf["remote"]["password"] - del homeconf["remote"]["username"] - self.updateValues(homeconf, self.config) - - except Exception, e: - print "Config Warning" - print_exc() - - - def parseConfig(self, config): - """parses a given configfile""" - - f = open(config) - - config = f.read() - - config = config.splitlines()[1:] - - conf = {} - - section, option, value, typ, desc = "", "", "", "", "" - - listmode = False - - for line in config: - comment = line.rfind("#") - if line.find(":", comment) < 0 > line.find("=", comment) and comment > 0 and line[comment - 1].isspace(): - line = line.rpartition("#") # removes comments - if line[1]: - line = line[0] - else: - line = line[2] - - line = line.strip() - - try: - if line == "": - continue - elif line.endswith(":"): - section, none, desc = line[:-1].partition('-') - section = section.strip() - desc = desc.replace('"', "").strip() - conf[section] = {"desc": desc} - else: - if listmode: - if line.endswith("]"): - listmode = False - line = line.replace("]", "") - - value += [self.cast(typ, x.strip()) for x in line.split(",") if x] - - if not listmode: - conf[section][option] = {"desc": desc, - "type": typ, - "value": value} - - - else: - content, none, value = line.partition("=") - - content, none, desc = content.partition(":") - - desc = desc.replace('"', "").strip() - - typ, none, option = content.strip().rpartition(" ") - - value = value.strip() - - if value.startswith("["): - if value.endswith("]"): - listmode = False - value = value[:-1] - else: - listmode = True - - value = [self.cast(typ, x.strip()) for x in value[1:].split(",") if x] - else: - value = self.cast(typ, value) - - if not listmode: - conf[section][option] = {"desc": desc, - "type": typ, - "value": value} - - except Exception, e: - print "Config Warning" - print_exc() - - f.close() - return conf - - - def updateValues(self, config, dest): - """sets the config values from a parsed config file to values in destination""" - - for section in config.iterkeys(): - if section in dest: - for option in config[section].iterkeys(): - if option in ("desc", "outline"): continue - - if option in dest[section]: - dest[section][option]["value"] = config[section][option]["value"] - - #else: - # dest[section][option] = config[section][option] - - - #else: - # dest[section] = config[section] - - def saveConfig(self, config, filename): - """saves config to filename""" - with open(filename, "wb") as f: - chmod(filename, 0600) - f.write("version: %i \n" % CONF_VERSION) - for section in config.iterkeys(): - f.write('\n%s - "%s":\n' % (section, config[section]["desc"])) - - for option, data in config[section].iteritems(): - if option in ("desc", "outline"): continue - - if isinstance(data["value"], list): - value = "[ \n" - for x in data["value"]: - value += "\t\t" + str(x) + ",\n" - value += "\t\t]\n" - else: - if type(data["value"]) in (str, unicode): - value = data["value"] + "\n" - else: - value = str(data["value"]) + "\n" - try: - f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value)) - except UnicodeEncodeError: - f.write('\t%s %s : "%s" = %s' % (data["type"], option, data["desc"], value.encode("utf8"))) - - def cast(self, typ, value): - """cast value to given format""" - if type(value) not in (str, unicode): - return value - - elif typ == "int": - return int(value) - elif typ == "bool": - return True if value.lower() in ("1", "true", "on", "an", "yes") else False - elif typ == "time": - if not value: value = "0:00" - if not ":" in value: value += ":00" - return value - elif typ in ("str", "file", "folder"): - try: - return value.encode("utf8") - except: - return value - else: - return value - - - def save(self): - """saves the configs to disk""" - - self.saveConfig(self.config, "pyload.conf") - self.saveConfig(self.plugin, "plugin.conf") - - - def __getitem__(self, section): - """provides dictonary like access: c['section']['option']""" - return Section(self, section) - - - def get(self, section, option): - """get value""" - val = self.config[section][option]["value"] - try: - if type(val) in (str, unicode): - return val.decode("utf8") - else: - return val - except: - return val - - def set(self, section, option, value): - """set value""" - - value = self.cast(self.config[section][option]["type"], value) - - self.config[section][option]["value"] = value - self.save() - - def getPlugin(self, plugin, option): - """gets a value for a plugin""" - val = self.plugin[plugin][option]["value"] - try: - if type(val) in (str, unicode): - return val.decode("utf8") - else: - return val - except: - return val - - def setPlugin(self, plugin, option, value): - """sets a value for a plugin""" - - value = self.cast(self.plugin[plugin][option]["type"], value) - - if self.pluginCB: self.pluginCB(plugin, option, value) - - self.plugin[plugin][option]["value"] = value - self.save() - - def getMetaData(self, section, option): - """ get all config data for an option """ - return self.config[section][option] - - def addPluginConfig(self, name, config, outline=""): - """adds config options with tuples (name, type, desc, default)""" - if name not in self.plugin: - conf = {"desc": name, - "outline": outline} - self.plugin[name] = conf - else: - conf = self.plugin[name] - conf["outline"] = outline - - for item in config: - if item[0] in conf: - conf[item[0]]["type"] = item[1] - conf[item[0]]["desc"] = item[2] - else: - conf[item[0]] = { - "desc": item[2], - "type": item[1], - "value": self.cast(item[1], item[3]) - } - - values = [x[0] for x in config] + ["desc", "outline"] - #delete old values - for item in conf.keys(): - if item not in values: - del conf[item] - - def deleteConfig(self, name): - """Removes a plugin config""" - if name in self.plugin: - del self.plugin[name] - - - def deleteOldPlugins(self): - """ remove old plugins from config """ - - for name in IGNORE: - if name in self.plugin: - del self.plugin[name] - - -class Section: - """provides dictionary like access for configparser""" - - def __init__(self, parser, section): - """Constructor""" - self.parser = parser - self.section = section - - def __getitem__(self, item): - """getitem""" - return self.parser.get(self.section, item) - - def __setitem__(self, item, value): - """setitem""" - self.parser.set(self.section, item, value) - - -if __name__ == "__main__": - pypath = "" - - from time import time - - a = time() - - c = ConfigParser() - - b = time() - - print "sec", b - a - - print c.config - - c.saveConfig(c.config, "user.conf") diff --git a/module/HookManager.py b/module/HookManager.py index 16f692d76..0ad37b321 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -15,51 +15,21 @@ along with this program; if not, see <http://www.gnu.org/licenses/>. @author: RaNaN, mkaay - @interface-version: 0.1 """ import __builtin__ -import traceback +from traceback import print_exc from thread import start_new_thread from threading import RLock from types import MethodType -from module.PluginThread import HookThread +from module.threads.HookThread import HookThread from module.plugins.PluginManager import literal_eval -from utils import lock +from utils import lock, to_string class HookManager: - """Manages hooks, delegates and handles Events. - - Every plugin can define events, \ - but some very usefull events are called by the Core. - Contrary to overwriting hook methods you can use event listener, - which provides additional entry point in the control flow. - Only do very short tasks or use threads. - - **Known Events:** - Most hook methods exists as events. These are the additional known events. - - ===================== ============== ================================== - Name Arguments Description - ===================== ============== ================================== - downloadPreparing fid A download was just queued and will be prepared now. - downloadStarts fid A plugin will immediately starts the download afterwards. - linksAdded links, pid Someone just added links, you are able to modify the links. - allDownloadsProcessed Every link was handled, pyload would idle afterwards. - allDownloadsFinished Every download in queue is finished. - unrarFinished folder, fname An Unrar job finished - configChanged The config was changed via the api. - pluginConfigChanged The plugin config changed, due to api or internal process. - ===================== ============== ================================== - - | Notes: - | allDownloadsProcessed is *always* called before allDownloadsFinished. - | configChanged is *always* called before pluginConfigChanged. - - - """ + """ Manages hooks, loading, unloading. """ def __init__(self, core): self.core = core @@ -68,34 +38,36 @@ class HookManager: __builtin__.hookManager = self #needed to let hooks register themself self.log = self.core.log - self.plugins = [] - self.pluginMap = {} - self.methods = {} #dict of names and list of methods usable by rpc - - self.events = {} # contains events - - #registering callback for config event - self.config.pluginCB = MethodType(self.dispatchEvent, "pluginConfigChanged", basestring) - - self.addEvent("pluginConfigChanged", self.manageHooks) + self.plugins = {} + self.methods = {} # dict of names and list of methods usable by rpc + self.events = {} # Contains event that will be registred self.lock = RLock() self.createIndex() - def try_catch(func): - def new(*args): - try: - return func(*args) - except Exception, e: - args[0].log.error(_("Error executing hooks: %s") % str(e)) - if args[0].core.debug: - traceback.print_exc() + #registering callback for config event + self.config.changeCB = MethodType(self.dispatchEvent, "configChanged", basestring) - return new + # manage hooks an config change + self.addEvent("configChanged", self.manageHooks) + @lock + def callInHooks(self, event, *args): + """ Calls a method in all hooks and catch / log errors""" + for plugin in self.plugins.itervalues(): + self.call(plugin, event, *args) + self.dispatchEvent(event, *args) + + def call(self, hook, f, *args): + try: + func = getattr(hook, f) + return func(*args) + except Exception, e: + hook.logError(_("Error when executing %s" % f), e) + if self.core.debug: + print_exc() def addRPC(self, plugin, func, doc): - plugin = plugin.rpartition(".")[2] doc = doc.strip() if doc else "" if plugin in self.methods: @@ -103,33 +75,30 @@ class HookManager: else: self.methods[plugin] = {func: doc} - def callRPC(self, plugin, func, args, parse): - if not args: args = tuple() - if parse: - args = tuple([literal_eval(x) for x in args]) + def callRPC(self, plugin, func, args): + if not args: args = [] + else: + args = literal_eval(args) - plugin = self.pluginMap[plugin] + plugin = self.plugins[plugin] f = getattr(plugin, func) return f(*args) - + @lock def createIndex(self): - plugins = [] - active = [] deactive = [] - for pluginname in self.core.pluginManager.hookPlugins: + for pluginname in self.core.pluginManager.getPlugins("hooks"): try: #hookClass = getattr(plugin, plugin.__name__) - if self.core.config.getPlugin(pluginname, "activated"): + if self.core.config.get(pluginname, "activated"): pluginClass = self.core.pluginManager.loadClass("hooks", pluginname) if not pluginClass: continue - + plugin = pluginClass(self.core, self) - plugins.append(plugin) - self.pluginMap[pluginClass.__name__] = plugin + self.plugins[pluginClass.__name__] = plugin if plugin.isActivated(): active.append(pluginClass.__name__) else: @@ -139,25 +108,26 @@ class HookManager: except: self.log.warning(_("Failed activating %(name)s") % {"name": pluginname}) if self.core.debug: - traceback.print_exc() + print_exc() self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active))) self.log.info(_("Deactivate plugins: %s") % ", ".join(sorted(deactive))) - self.plugins = plugins - def manageHooks(self, plugin, name, value): + # check if section was a plugin + if plugin not in self.core.pluginManager.getPlugins("hooks"): + return + if name == "activated" and value: self.activateHook(plugin) elif name == "activated" and not value: self.deactivateHook(plugin) + @lock def activateHook(self, plugin): - #check if already loaded - for inst in self.plugins: - if inst.__name__ == plugin: - return + if plugin in self.plugins: + return pluginClass = self.core.pluginManager.loadClass("hooks", plugin) @@ -166,150 +136,109 @@ class HookManager: self.log.debug("Plugin loaded: %s" % plugin) plugin = pluginClass(self.core, self) - self.plugins.append(plugin) - self.pluginMap[pluginClass.__name__] = plugin + self.plugins[pluginClass.__name__] = plugin - # call core Ready - start_new_thread(plugin.coreReady, tuple()) + # active the hook in new thread + start_new_thread(plugin.activate, tuple()) + self.registerEvents() + @lock def deactivateHook(self, plugin): + if plugin not in self.plugins: + return + else: + hook = self.plugins[plugin] - hook = None - for inst in self.plugins: - if inst.__name__ == plugin: - hook = inst - - if not hook: return - - self.log.debug("Plugin unloaded: %s" % plugin) - - hook.unload() + self.call(hook, "deactivate") + self.log.debug("Plugin deactivated: %s" % plugin) #remove periodic call self.log.debug("Removed callback %s" % self.core.scheduler.removeJob(hook.cb)) - self.plugins.remove(hook) - del self.pluginMap[hook.__name__] + del self.plugins[hook.__name__] + #remove event listener + for f in dir(hook): + if f.startswith("__") or type(getattr(hook, f)) != MethodType: + continue + self.core.eventManager.removeFromEvents(getattr(hook, f)) - @try_catch - def coreReady(self): - for plugin in self.plugins: + def activateHooks(self): + self.log.info(_("Activating Plugins...")) + for plugin in self.plugins.itervalues(): if plugin.isActivated(): - plugin.coreReady() - - self.dispatchEvent("coreReady") + self.call(plugin, "activate") - @try_catch - def coreExiting(self): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.coreExiting() + self.registerEvents() - self.dispatchEvent("coreExiting") + def deactivateHooks(self): + """ Called when core is shutting down """ + self.log.info(_("Deactivating Plugins...")) + for plugin in self.plugins.itervalues(): + self.call(plugin, "deactivate") - @lock def downloadPreparing(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.downloadPreparing(pyfile) - - self.dispatchEvent("downloadPreparing", pyfile) + self.callInHooks("downloadPreparing", pyfile) - @lock def downloadFinished(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - if "downloadFinished" in plugin.__threaded__: - self.startThread(plugin.downloadFinished, pyfile) - else: - plugin.downloadFinished(pyfile) + self.callInHooks("downloadFinished", pyfile) - self.dispatchEvent("downloadFinished", pyfile) - - @lock - @try_catch def downloadFailed(self, pyfile): - for plugin in self.plugins: - if plugin.isActivated(): - if "downloadFailed" in plugin.__threaded__: - self.startThread(plugin.downloadFinished, pyfile) - else: - plugin.downloadFailed(pyfile) + self.callInHooks("downloadFailed", pyfile) - self.dispatchEvent("downloadFailed", pyfile) - - @lock def packageFinished(self, package): - for plugin in self.plugins: - if plugin.isActivated(): - if "packageFinished" in plugin.__threaded__: - self.startThread(plugin.packageFinished, package) - else: - plugin.packageFinished(package) - - self.dispatchEvent("packageFinished", package) + self.callInHooks("packageFinished", package) - @lock def beforeReconnecting(self, ip): - for plugin in self.plugins: - plugin.beforeReconnecting(ip) - - self.dispatchEvent("beforeReconnecting", ip) + self.callInHooks("beforeReconnecting", ip) - @lock def afterReconnecting(self, ip): - for plugin in self.plugins: - if plugin.isActivated(): - plugin.afterReconnecting(ip) - - self.dispatchEvent("afterReconnecting", ip) + self.callInHooks("afterReconnecting", ip) + @lock def startThread(self, function, *args, **kwargs): - t = HookThread(self.core.threadManager, function, args, kwargs) + HookThread(self.core.threadManager, function, args, kwargs) def activePlugins(self): """ returns all active plugins """ - return [x for x in self.plugins if x.isActivated()] + return [x for x in self.plugins.itervalues() if x.isActivated()] def getAllInfo(self): """returns info stored by hook plugins""" info = {} - for name, plugin in self.pluginMap.iteritems(): + for name, plugin in self.plugins.iteritems(): if plugin.info: #copy and convert so str - info[name] = dict([(x, str(y) if not isinstance(y, basestring) else y) for x, y in plugin.info.iteritems()]) + info[name] = dict( + [(x, to_string(y)) for x, y in plugin.info.iteritems()]) return info - def getInfo(self, plugin): info = {} - if plugin in self.pluginMap and self.pluginMap[plugin].info: - info = dict([(x, str(y) if not isinstance(y, basestring) else y) - for x, y in self.pluginMap[plugin].info.iteritems()]) + if plugin in self.plugins and self.plugins[plugin].info: + info = dict([(x, to_string(y)) + for x, y in self.plugins[plugin].info.iteritems()]) return info - def addEvent(self, event, func): - """Adds an event listener for event name""" - if event in self.events: - self.events[event].append(func) - else: - self.events[event] = [func] - - def removeEvent(self, event, func): - """removes previously added event listener""" - if event in self.events: - self.events[event].remove(func) - - def dispatchEvent(self, event, *args): - """dispatches event with args""" - if event in self.events: - for f in self.events[event]: - try: - f(*args) - except Exception, e: - self.log.warning("Error calling event handler %s: %s, %s, %s" - % (event, f, args, str(e))) - if self.core.debug: - traceback.print_exc() + def addEventListener(self, plugin, func, event): + if plugin not in self.events: + self.events[plugin] = [] + self.events[plugin].append((func, event)) + + def registerEvents(self): + for name, plugin in self.plugins.iteritems(): + if name in self.events: + for func, event in self.events[name]: + self.addEvent(event, getattr(plugin, func)) + # clean up + del self.events[name] + + def addConfigHandler(self, plugin, func): + pass #TODO + + def addEvent(self, *args): + self.core.eventManager.addEvent(*args) + + def dispatchEvent(self, *args): + self.core.eventManager.dispatchEvent(*args) diff --git a/module/InitHomeDir.py b/module/InitHomeDir.py index 156c9f932..14207847e 100644 --- a/module/InitHomeDir.py +++ b/module/InitHomeDir.py @@ -52,7 +52,7 @@ else: __builtin__.homedir = homedir -args = " ".join(argv[1:]) +args = " ".join(argv) # dirty method to set configdir from commandline arguments if "--configdir=" in args: @@ -63,6 +63,10 @@ if "--configdir=" in args: configdir = args[pos + 12:].strip() else: configdir = args[pos + 12:end].strip() +elif "nosetests" in args: + print "Running in test mode" + configdir = join(pypath, "tests", "config") + elif path.exists(path.join(pypath, "module", "config", "configdir")): f = open(path.join(pypath, "module", "config", "configdir"), "rb") c = f.read().strip() diff --git a/module/PluginThread.py b/module/PluginThread.py deleted file mode 100644 index 56c36c778..000000000 --- a/module/PluginThread.py +++ /dev/null @@ -1,677 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -from Queue import Queue -from threading import Thread -from os import listdir, stat -from os.path import join -from time import sleep, time, strftime, gmtime -from traceback import print_exc, format_exc -from pprint import pformat -from sys import exc_info, exc_clear -from copy import copy -from types import MethodType - -from pycurl import error - -from PyFile import PyFile -from plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload -from common.packagetools import parseNames -from utils import save_join -from Api import OnlineStatus - -class PluginThread(Thread): - """abstract base class for thread types""" - - #---------------------------------------------------------------------- - def __init__(self, manager): - """Constructor""" - Thread.__init__(self) - self.setDaemon(True) - self.m = manager #thread manager - - - def writeDebugReport(self, pyfile): - """ writes a - :return: - """ - - dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) - dump = self.getDebugDump(pyfile) - - try: - import zipfile - - zip = zipfile.ZipFile(dump_name, "w") - - for f in listdir(join("tmp", pyfile.pluginname)): - try: - # avoid encoding errors - zip.write(join("tmp", pyfile.pluginname, f), save_join(pyfile.pluginname, f)) - except: - pass - - info = zipfile.ZipInfo(save_join(pyfile.pluginname, "debug_Report.txt"), gmtime()) - info.external_attr = 0644 << 16L # change permissions - - zip.writestr(info, dump) - zip.close() - - if not stat(dump_name).st_size: - raise Exception("Empty Zipfile") - - except Exception, e: - self.m.log.debug("Error creating zip file: %s" % e) - - dump_name = dump_name.replace(".zip", ".txt") - f = open(dump_name, "wb") - f.write(dump) - f.close() - - self.m.core.log.info("Debug Report written to %s" % dump_name) - - def getDebugDump(self, pyfile): - dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( - self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version__, format_exc()) - - tb = exc_info()[2] - stack = [] - while tb: - stack.append(tb.tb_frame) - tb = tb.tb_next - - for frame in stack[1:]: - dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, - frame.f_code.co_filename, - frame.f_lineno) - - for key, value in frame.f_locals.items(): - dump += "\t%20s = " % key - try: - dump += pformat(value) + "\n" - except Exception, e: - dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" - - del frame - - del stack #delete it just to be sure... - - dump += "\n\nPLUGIN OBJECT DUMP: \n\n" - - for name in dir(pyfile.plugin): - attr = getattr(pyfile.plugin, name) - if not name.endswith("__") and type(attr) != MethodType: - dump += "\t%20s = " % name - try: - dump += pformat(attr) + "\n" - except Exception, e: - dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" - - dump += "\nPYFILE OBJECT DUMP: \n\n" - - for name in dir(pyfile): - attr = getattr(pyfile, name) - if not name.endswith("__") and type(attr) != MethodType: - dump += "\t%20s = " % name - try: - dump += pformat(attr) + "\n" - except Exception, e: - dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" - - if pyfile.pluginname in self.m.core.config.plugin: - dump += "\n\nCONFIG: \n\n" - dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" - - return dump - - def clean(self, pyfile): - """ set thread unactive and release pyfile """ - self.active = False - pyfile.release() - - -class DownloadThread(PluginThread): - """thread for downloading files from 'real' hoster plugins""" - - #---------------------------------------------------------------------- - def __init__(self, manager): - """Constructor""" - PluginThread.__init__(self, manager) - - self.queue = Queue() # job queue - self.active = False - - self.start() - - #---------------------------------------------------------------------- - def run(self): - """run method""" - pyfile = None - - while True: - del pyfile - self.active = self.queue.get() - pyfile = self.active - - if self.active == "quit": - self.active = False - self.m.threads.remove(self) - return True - - try: - if not pyfile.hasPlugin(): continue - #this pyfile was deleted while queueing - - pyfile.plugin.checkForSameFiles(starting=True) - self.m.log.info(_("Download starts: %s" % pyfile.name)) - - # start download - self.m.core.hookManager.downloadPreparing(pyfile) - pyfile.plugin.preprocessing(self) - - self.m.log.info(_("Download finished: %s") % pyfile.name) - self.m.core.hookManager.downloadFinished(pyfile) - self.m.core.files.checkPackageFinished(pyfile) - - except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) - pyfile.setStatus("failed") - pyfile.error = "Plugin does not work" - self.clean(pyfile) - continue - - except Abort: - try: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - except: - pass - - pyfile.setStatus("aborted") - - self.clean(pyfile) - continue - - except Reconnect: - self.queue.put(pyfile) - #pyfile.req.clearCookies() - - while self.m.reconnecting.isSet(): - sleep(0.5) - - continue - - except Retry, e: - reason = e.args[0] - self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) - self.queue.put(pyfile) - continue - - except Fail, e: - msg = e.args[0] - - if msg == "offline": - pyfile.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % pyfile.name) - elif msg == "temp. offline": - pyfile.setStatus("temp. offline") - self.m.log.warning(_("Download is temporary offline: %s") % pyfile.name) - else: - pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) - pyfile.error = msg - - self.m.core.hookManager.downloadFailed(pyfile) - self.clean(pyfile) - continue - - except error, e: - if len(e.args) == 2: - code, msg = e.args - else: - code = 0 - msg = e.args - - self.m.log.debug("pycurl exception %s: %s" % (code, msg)) - - if code in (7, 18, 28, 52, 56): - self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) - wait = time() + 60 - - pyfile.waitUntil = wait - pyfile.setStatus("waiting") - while time() < wait: - sleep(1) - if pyfile.abort: - break - - if pyfile.abort: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - pyfile.setStatus("aborted") - - self.clean(pyfile) - else: - self.queue.put(pyfile) - - continue - - else: - pyfile.setStatus("failed") - self.m.log.error("pycurl error %s: %s" % (code, msg)) - if self.m.core.debug: - print_exc() - self.writeDebugReport(pyfile) - - self.m.core.hookManager.downloadFailed(pyfile) - - self.clean(pyfile) - continue - - except SkipDownload, e: - pyfile.setStatus("skipped") - - self.m.log.info( - _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) - - self.clean(pyfile) - - self.m.core.files.checkPackageFinished(pyfile) - - self.active = False - self.m.core.files.save() - - continue - - - except Exception, e: - pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) - pyfile.error = str(e) - - if self.m.core.debug: - print_exc() - self.writeDebugReport(pyfile) - - self.m.core.hookManager.downloadFailed(pyfile) - self.clean(pyfile) - continue - - finally: - self.m.core.files.save() - pyfile.checkIfProcessed() - exc_clear() - - - #pyfile.plugin.req.clean() - - self.active = False - pyfile.finishIfDone() - self.m.core.files.save() - - - def put(self, job): - """assing job to thread""" - self.queue.put(job) - - - def stop(self): - """stops the thread""" - self.put("quit") - - -class DecrypterThread(PluginThread): - """thread for decrypting""" - - def __init__(self, manager, pyfile): - """constructor""" - PluginThread.__init__(self, manager) - - self.active = pyfile - manager.localThreads.append(self) - - pyfile.setStatus("decrypting") - - self.start() - - def getActiveFiles(self): - return [self.active] - - def run(self): - """run method""" - - pyfile = self.active - retry = False - - try: - self.m.log.info(_("Decrypting starts: %s") % self.active.name) - self.active.plugin.preprocessing(self) - - except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) - return - - except Fail, e: - msg = e.args[0] - - if msg == "offline": - self.active.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % self.active.name) - else: - self.active.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": msg}) - self.active.error = msg - - return - - except Abort: - self.m.log.info(_("Download aborted: %s") % pyfile.name) - pyfile.setStatus("aborted") - - return - - except Retry: - self.m.log.info(_("Retrying %s") % self.active.name) - retry = True - return self.run() - - except Exception, e: - self.active.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": str(e)}) - self.active.error = str(e) - - if self.m.core.debug: - print_exc() - self.writeDebugReport(pyfile) - - return - - - finally: - if not retry: - self.active.release() - self.active = False - self.m.core.files.save() - self.m.localThreads.remove(self) - exc_clear() - - - #self.m.core.hookManager.downloadFinished(pyfile) - - - #self.m.localThreads.remove(self) - #self.active.finishIfDone() - if not retry: - pyfile.delete() - - -class HookThread(PluginThread): - """thread for hooks""" - - #---------------------------------------------------------------------- - def __init__(self, m, function, args, kwargs): - """Constructor""" - PluginThread.__init__(self, m) - - self.f = function - self.args = args - self.kwargs = kwargs - - self.active = [] - - m.localThreads.append(self) - - self.start() - - def getActiveFiles(self): - return self.active - - def addActive(self, pyfile): - """ Adds a pyfile to active list and thus will be displayed on overview""" - if pyfile not in self.active: - self.active.append(pyfile) - - def finishFile(self, pyfile): - if pyfile in self.active: - self.active.remove(pyfile) - - pyfile.finishIfDone() - - def run(self): - try: - try: - self.kwargs["thread"] = self - self.f(*self.args, **self.kwargs) - except TypeError, e: - #dirty method to filter out exceptions - if "unexpected keyword argument 'thread'" not in e.args[0]: - raise - - del self.kwargs["thread"] - self.f(*self.args, **self.kwargs) - finally: - local = copy(self.active) - for x in local: - self.finishFile(x) - - self.m.localThreads.remove(self) - - -class InfoThread(PluginThread): - def __init__(self, manager, data, pid=-1, rid=-1, add=False): - """Constructor""" - PluginThread.__init__(self, manager) - - self.data = data - self.pid = pid # package id - # [ .. (name, plugin) .. ] - - self.rid = rid #result id - self.add = add #add packages instead of return result - - self.cache = [] #accumulated data - - self.start() - - def run(self): - """run method""" - - plugins = {} - container = [] - - for url, plugin in self.data: - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] - - - # filter out container plugins - for name in self.m.core.pluginManager.containerPlugins: - if name in plugins: - container.extend([(name, url) for url in plugins[name]]) - - del plugins[name] - - #directly write to database - if self.pid > -1: - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) - self.m.core.files.save() - - elif self.add: - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) - - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateCache(pluginname, result) - - packs = parseNames([(name, url) for name, x, y, url in self.cache]) - - self.m.log.debug("Fetched and generated %d packages" % len(packs)) - - for k, v in packs: - self.m.core.api.addPackage(k, v) - - #empty cache - del self.cache[:] - - else: #post the results - - - for name, url in container: - #attach container content - try: - data = self.decryptContainer(name, url) - except: - print_exc() - self.m.log.error("Could not decrypt container.") - data = [] - - for url, plugin in data: - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] - - self.m.infoResults[self.rid] = {} - - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - - #force to process cache - if self.cache: - self.updateResult(pluginname, [], True) - - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateResult(pluginname, result, True) - - self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} - - self.m.timestamp = time() + 5 * 60 - - - def updateDB(self, plugin, result): - self.m.core.files.updateFileInfo(result, self.pid) - - def updateResult(self, plugin, result, force=False): - #parse package name and generate result - #accumulate results - - self.cache.extend(result) - - if len(self.cache) >= 20 or force: - #used for package generating - tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) - for name, size, status, url in self.cache] - - data = parseNames(tmp) - result = {} - for k, v in data.iteritems(): - for url, status in v: - status.packagename = k - result[url] = status - - self.m.setInfoResults(self.rid, result) - - self.cache = [] - - def updateCache(self, plugin, result): - self.cache.extend(result) - - def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): - try: - result = [] #result loaded from cache - process = [] #urls to process - for url in urls: - if url in self.m.infoCache: - result.append(self.m.infoCache[url]) - else: - process.append(url) - - if result: - self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) - cb(pluginname, result) - - if process: - self.m.log.debug("Run Info Fetching for %s" % pluginname) - for result in plugin.getInfo(process): - #result = [ .. (name, size, status, url) .. ] - if not type(result) == list: result = [result] - - for res in result: - self.m.infoCache[res[3]] = res - - cb(pluginname, result) - - self.m.log.debug("Finished Info Fetching for %s" % pluginname) - except Exception, e: - self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % - {"name": pluginname, "err": str(e)}) - if self.m.core.debug: - print_exc() - - # generate default results - if err: - result = [(url, 0, 3, url) for url in urls] - cb(pluginname, result) - - - def decryptContainer(self, plugin, url): - data = [] - # only works on container plugins - - self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) - - # dummy pyfile - pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) - - pyfile.initPlugin() - - # little plugin lifecycle - try: - pyfile.plugin.setup() - pyfile.plugin.loadToDisk() - pyfile.plugin.decrypt(pyfile) - pyfile.plugin.deleteTmp() - - for pack in pyfile.plugin.packages: - pyfile.plugin.urls.extend(pack[1]) - - data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) - - self.m.log.debug("Got %d links." % len(data)) - - except Exception, e: - self.m.log.debug("Pre decrypting error: %s" % str(e)) - finally: - pyfile.release() - - return data diff --git a/module/PullEvents.py b/module/PullEvents.py deleted file mode 100644 index 5ec76765e..000000000 --- a/module/PullEvents.py +++ /dev/null @@ -1,120 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from time import time -from module.utils import uniqify - -class PullManager(): - def __init__(self, core): - self.core = core - self.clients = [] - - def newClient(self, uuid): - self.clients.append(Client(uuid)) - - def clean(self): - for n, client in enumerate(self.clients): - if client.lastActive + 30 < time(): - del self.clients[n] - - def getEvents(self, uuid): - events = [] - validUuid = False - for client in self.clients: - if client.uuid == uuid: - client.lastActive = time() - validUuid = True - while client.newEvents(): - events.append(client.popEvent().toList()) - break - if not validUuid: - self.newClient(uuid) - events = [ReloadAllEvent("queue").toList(), ReloadAllEvent("collector").toList()] - return uniqify(events, repr) - - def addEvent(self, event): - for client in self.clients: - client.addEvent(event) - -class Client(): - def __init__(self, uuid): - self.uuid = uuid - self.lastActive = time() - self.events = [] - - def newEvents(self): - return len(self.events) > 0 - - def popEvent(self): - if not len(self.events): - return None - return self.events.pop(0) - - def addEvent(self, event): - self.events.append(event) - -class UpdateEvent(): - def __init__(self, itype, iid, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.destination = destination - - def toList(self): - return ["update", self.destination, self.type, self.id] - -class RemoveEvent(): - def __init__(self, itype, iid, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.destination = destination - - def toList(self): - return ["remove", self.destination, self.type, self.id] - -class InsertEvent(): - def __init__(self, itype, iid, after, destination): - assert itype == "pack" or itype == "file" - assert destination == "queue" or destination == "collector" - self.type = itype - self.id = iid - self.after = after - self.destination = destination - - def toList(self): - return ["insert", self.destination, self.type, self.id, self.after] - -class ReloadAllEvent(): - def __init__(self, destination): - assert destination == "queue" or destination == "collector" - self.destination = destination - - def toList(self): - return ["reload", self.destination] - -class AccountUpdateEvent(): - def toList(self): - return ["account"] - -class ConfigUpdateEvent(): - def toList(self): - return ["config"] diff --git a/module/PyFile.py b/module/PyFile.py index 3dede9360..4f8b95124 100644 --- a/module/PyFile.py +++ b/module/PyFile.py @@ -17,13 +17,12 @@ @author: mkaay """ -from module.PullEvents import UpdateEvent -from module.utils import formatSize, lock from time import sleep, time - from threading import RLock +from module.utils import formatSize, lock + statusMap = { "finished": 0, "offline": 1, @@ -50,16 +49,16 @@ class PyFile(object): """ Represents a file object at runtime """ - __slots__ = ("m", "id", "url", "name", "size", "_size", "status", "pluginname", "packageid", + __slots__ = ("m", "id", "url", "_name", "name", "size", "_size", "status", "pluginname", "packageid", "error", "order", "lock", "plugin", "waitUntil", "active", "abort", "statusname", - "reconnected", "progress", "maxprogress", "pluginmodule", "pluginclass") + "reconnected", "progress", "maxprogress", "pluginclass") def __init__(self, manager, id, url, name, size, status, error, pluginname, package, order): self.m = manager self.id = int(id) self.url = url - self.name = name + self._name = name self.size = size self.status = status self.pluginname = pluginname @@ -90,16 +89,33 @@ class PyFile(object): # will convert all sizes to ints size = property(lambda self: self._size, setSize) - + + def getName(self): + try: + if self.plugin.req.name: + return self.plugin.req.name + else: + return self._name + except: + return self._name + + def setName(self, name): + """ Only set unicode or utf8 strings as name """ + if type(name) == str: + name = name.decode("utf8") + + self._name = name + + name = property(getName, setName) + def __repr__(self): - return "PyFile %s: %s@%s" % (self.id, self.name, self.pluginname) + return "<PyFile %s: %s@%s>" % (self.id, self.name, self.pluginname) @lock def initPlugin(self): """ inits plugin instance """ if not self.plugin: - self.pluginmodule = self.m.core.pluginManager.getPlugin(self.pluginname) - self.pluginclass = getattr(self.pluginmodule, self.m.core.pluginManager.getPluginName(self.pluginname)) + self.pluginclass = self.m.core.pluginManager.getPlugin(self.pluginname) self.plugin = self.pluginclass(self) @lock @@ -276,8 +292,7 @@ class PyFile(object): return self.size def notifyChange(self): - e = UpdateEvent("file", self.id, "collector" if not self.package().queue else "queue") - self.m.core.pullManager.addEvent(e) + self.m.core.eventManager.dispatchEvent("linkUpdated", self.id, self.packageid) def setProgress(self, value): if not value == self.progress: diff --git a/module/PyPackage.py b/module/PyPackage.py index f3be6c886..970982e68 100644 --- a/module/PyPackage.py +++ b/module/PyPackage.py @@ -17,9 +17,6 @@ @author: mkaay """ -from module.PullEvents import UpdateEvent -from module.utils import save_path - class PyPackage(): """ Represents a package object at runtime @@ -30,17 +27,13 @@ class PyPackage(): self.id = int(id) self.name = name - self._folder = folder + self.folder = folder self.site = site self.password = password self.queue = queue self.order = order self.setFinished = False - @property - def folder(self): - return save_path(self._folder) - def toDict(self): """ Returns a dictionary representation of the data. @@ -74,7 +67,13 @@ class PyPackage(): def delete(self): self.m.deletePackage(self.id) - + + def deleteIfEmpty(self): + """ True if deleted """ + if not len(self.getChildren()): + self.delete() + return True + return False + def notifyChange(self): - e = UpdateEvent("pack", self.id, "collector" if not self.queue else "queue") - self.m.core.pullManager.addEvent(e) + self.m.core.eventManager.dispatchEvent("packageUpdated", self.id) diff --git a/module/common/APIExerciser.py b/module/common/APIExerciser.py index 96f5ce9cf..657e83c78 100644 --- a/module/common/APIExerciser.py +++ b/module/common/APIExerciser.py @@ -114,7 +114,7 @@ class APIExerciser(Thread): name = "".join(sample(string.ascii_letters, 10)) urls = createURLs() - self.api.addPackage(name, urls, choice([Destination.Queue, Destination.Collector])) + self.api.addPackage(name, urls, choice([Destination.Queue, Destination.Collector]), "") def deleteFiles(self): @@ -154,4 +154,4 @@ class APIExerciser(Thread): self.api.getAccounts(False) def getCaptchaTask(self): - self.api.getCaptchaTask(False)
\ No newline at end of file + self.api.getCaptchaTask(False) diff --git a/module/config/ConfigParser.py b/module/config/ConfigParser.py new file mode 100644 index 000000000..a9e74dd20 --- /dev/null +++ b/module/config/ConfigParser.py @@ -0,0 +1,253 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement +from time import sleep +from os.path import exists +from gettext import gettext +from new_collections import namedtuple, OrderedDict + +from module.utils import from_string +from module.utils.fs import chmod + +from default import make_config + +CONF_VERSION = 2 +SectionTuple = namedtuple("SectionTuple", "name description long_desc config") +ConfigData = namedtuple("ConfigData", "name type description default") + +class ConfigParser: + """ + Holds and manage the configuration + meta data. + Actually only the values are read from disk, all meta data have to be provided first via addConfigSection. + """ + + CONFIG = "pyload.conf" + PLUGIN = "plugin.conf" + + def __init__(self): + """Constructor""" + + # core config sections from pyload + self.baseSections = [] + + # Meta data information + self.config = OrderedDict() + # The actual config values + self.values = {} + + self.changeCB = None # callback when config value was changed + + self.checkVersion() + + self.loadDefault() + self.parseValues(self.CONFIG) + + def loadDefault(self): + make_config(self) + + def checkVersion(self): + """Determines if config need to be deleted""" + e = None + # workaround conflict, with GUI (which also access the config) so try read in 3 times + for i in range(0, 3): + try: + for conf in (self.CONFIG, self.PLUGIN): + if exists(conf): + f = open(conf, "rb") + v = f.readline() + f.close() + v = v[v.find(":") + 1:].strip() + + if not v or int(v) < CONF_VERSION: + f = open(conf, "wb") + f.write("version: " + str(CONF_VERSION)) + f.close() + print "Old version of %s deleted" % conf + else: + f = open(conf, "wb") + f.write("version:" + str(CONF_VERSION)) + f.close() + + except Exception, ex: + e = ex + sleep(0.3) + if e: raise e + + + def parseValues(self, filename): + """read config values from file""" + f = open(filename, "rb") + config = f.readlines()[1:] + + # save the current section + section = "" + + for line in config: + line = line.strip() + + # comment line, different variants + if not line or line.startswith("#") or line.startswith("//") or line.startswith(";"): continue + + if line.startswith("["): + section = line.replace("[", "").replace("]", "") + + if section not in self.config: + print "Unrecognzied section", section + section = "" + + else: + name, non, value = line.rpartition("=") + name = name.strip() + value = value.strip() + + if not section: + print "Value without section", name + continue + + if name in self.config[section].config: + self.set(section, name, value, sync=False) + else: + print "Unrecognized option", section, name + + + def save(self): + """saves config to filename""" + + # seperate pyload and plugin conf + configs = [] + for c in (self.CONFIG, self.PLUGIN): + f = open(c, "wb") + configs.append(f) + chmod(c, 0600) + f.write("version: %i\n\n" % CONF_VERSION) + + + # write on 2 files + for section, data in self.config.iteritems(): + f = configs[0] if section in self.baseSections else configs[1] + + f.write("[%s]\n" % section) + + for option, data in data.config.iteritems(): + value = self.get(section, option) + if type(value) == unicode: value = value.encode("utf8") + else: value = str(value) + + f.write('%s = %s\n' % (option, value)) + + f.write("\n") + + [f.close() for f in configs] + + def __getitem__(self, section): + """provides dictonary like access: c['section']['option']""" + return Section(self, section) + + def get(self, section, option): + """get value""" + if option in self.values[section]: + return self.values[section][option] + else: + return self.config[section].config[option].default + + def set(self, section, option, value, sync=True): + """set value""" + + data = self.config[section].config[option] + value = from_string(value, data.type) + + # only save when different to defaul values + if value != data.default or (option in self.values[section] and value != self.values[section][option]): + self.values[section][option] = value + if sync: + if self.changeCB: self.changeCB(section, option, value) + self.save() + + def getPlugin(self, *args): + """gets a value for a plugin""" + ret = self.get(*args) + print "Deprecated method getPlugin%s -> %s" % (str(args), ret) + return ret + + def setPlugin(self, *args): + """sets a value for a plugin""" + print "Deprecated method setPlugin%s" % str(args) + self.set(*args) + + def getMetaData(self, section, option): + """ get all config data for an option """ + return self.config[section].config[option] + + def getBaseSections(self): + for section, data in self.config.iteritems(): + if section in self.baseSections: + yield section, data + return + + def getPluginSections(self): + for section, data in self.config.iteritems(): + if section not in self.baseSections: + yield section, data + return + + def addConfigSection(self, section, name, desc, long_desc, config, base=False): + """Adds a section to the config. `config` is a list of config tuples as used in plugin api definied as: + Either (name, type, verbose_name, default_value) or + (name, type, verbose_name, short_description, default_value) + The ordner of the config elements are preserved with OrdererDict + """ + d = OrderedDict() + + for entry in config: + if len(entry) == 5: + conf_name, type, conf_desc, conf_verbose, default = entry + else: # config options without tooltip / description + conf_name, type, conf_desc, default = entry + conf_verbose = "" + + d[conf_name] = ConfigData(gettext(conf_desc), type, gettext(conf_verbose), from_string(default, type)) + + if base: + if section not in self.baseSections: self.baseSections.append(section) + elif section in self.baseSections: + return # would overwrite base section + + data = SectionTuple(gettext(name), gettext(desc), gettext(long_desc), d) + self.config[section] = data + + if section not in self.values: + self.values[section] = {} + +class Section: + """provides dictionary like access for configparser""" + + def __init__(self, parser, section): + """Constructor""" + self.parser = parser + self.section = section + + def __getitem__(self, item): + """getitem""" + return self.parser.get(self.section, item) + + def __setitem__(self, item, value): + """setitem""" + self.parser.set(self.section, item, value) + + +if __name__ == "__main__": + pypath = "" + + from time import time + + a = time() + + c = ConfigParser() + + b = time() + + print "sec", b - a + + print c.config + + c.saveConfig(c.config, "user.conf") diff --git a/module/config/__init__.py b/module/config/__init__.py new file mode 100644 index 000000000..4b31e848b --- /dev/null +++ b/module/config/__init__.py @@ -0,0 +1 @@ +__author__ = 'christian' diff --git a/module/config/default.conf b/module/config/default.conf deleted file mode 100644 index 335ca10fe..000000000 --- a/module/config/default.conf +++ /dev/null @@ -1,65 +0,0 @@ -version: 1
-
-remote - "Remote":
- int port : "Port" = 7227
- ip listenaddr : "Adress" = 0.0.0.0
- bool nolocalauth : "No authentication on local connections" = True
- bool activated : "Activated" = True
-ssl - "SSL":
- bool activated : "Activated"= False
- file cert : "SSL Certificate" = ssl.crt
- file key : "SSL Key" = ssl.key
-webinterface - "Webinterface":
- bool activated : "Activated" = True
- builtin;threaded;fastcgi;lightweight server : "Server" = builtin
- bool https : "Use HTTPS" = False
- ip host : "IP" = 0.0.0.0
- int port : "Port" = 8001
- str template : "Template" = default
- str prefix: "Path Prefix" =
-log - "Log":
- bool file_log : "File Log" = True
- folder log_folder : "Folder" = Logs
- int log_count : "Count" = 5
- int log_size : "Size in kb" = 100
- bool log_rotate : "Log Rotate" = True
-general - "General":
- en;de;fr;it;es;nl;sv;ru;pl;cs;sr;pt_BR language : "Language" = en
- folder download_folder : "Download Folder" = Downloads
- bool debug_mode : "Debug Mode" = False
- bool checksum : "Use Checksum" = False
- int min_free_space : "Min Free Space (MB)" = 200
- bool folder_per_package : "Create folder for each package" = True
- int renice : "CPU Priority" = 0
-download - "Download":
- int chunks : "Max connections for one download" = 3
- int max_downloads : "Max Parallel Downloads" = 3
- int max_speed : "Max Download Speed in kb/s" = -1
- bool limit_speed : "Limit Download Speed" = False
- str interface : "Download interface to bind (ip or Name)" = None
- bool ipv6 : "Allow IPv6" = False
- bool skip_existing : "Skip already existing files" = False
-permission - "Permissions":
- bool change_user : "Change user of running process" = False
- str user : "Username" = user
- str folder : "Folder Permission mode" = 0755
- bool change_file : "Change file mode of downloads" = False
- str file : "Filemode for Downloads" = 0644
- bool change_group : "Change group of running process" = False
- str group : "Groupname" = users
- bool change_dl : "Change Group and User of Downloads" = False
-reconnect - "Reconnect":
- bool activated : "Use Reconnect" = False
- str method : "Method" = None
- time startTime : "Start" = 0:00
- time endTime : "End" = 0:00
-downloadTime - "Download Time":
- time start : "Start" = 0:00
- time end : "End" = 0:00
-proxy - "Proxy":
- str address : "Address" = "localhost"
- int port : "Port" = 7070
- http;socks4;socks5 type : "Protocol" = http
- str username : "Username" = None
- password password : "Password" = None
- bool proxy : "Use Proxy" = False
diff --git a/module/config/default.py b/module/config/default.py new file mode 100644 index 000000000..1dbb58eca --- /dev/null +++ b/module/config/default.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- + +""" +Configuration layout for default base config +""" + +#TODO: write tooltips and descriptions + +def make_config(config): + # Check if gettext is installed + _ = lambda x: x + + config.addConfigSection("remote", _("Remote"), _("Description"), _("Long description"), + [ + ("nolocalauth", "bool", _("No authentication on local connections"), _("Tooltip"), True), + ("activated", "bool", _("Activated"), _("Tooltip"), True), + ("port", "int", _("Port"), _("Tooltip"), 7227), + ("listenaddr", "ip", _("Adress"), _("Tooltip"), "0.0.0.0"), + ], + True) + + config.addConfigSection("log", _("Log"), _("Description"), _("Long description"), + [ + ("log_size", "int", _("Size in kb"), _("Tooltip"), 100), + ("log_folder", "folder", _("Folder"), _("Tooltip"), "Logs"), + ("file_log", "bool", _("File Log"), _("Tooltip"), True), + ("log_count", "int", _("Count"), _("Tooltip"), 5), + ("log_rotate", "bool", _("Log Rotate"), _("Tooltip"), True), + ], + True) + + config.addConfigSection("permission", _("Permissions"), _("Description"), _("Long description"), + [ + ("group", "str", _("Groupname"), _("Tooltip"), "users"), + ("change_dl", "bool", _("Change Group and User of Downloads"), _("Tooltip"), False), + ("change_file", "bool", _("Change file mode of downloads"), _("Tooltip"), False), + ("user", "str", _("Username"), _("Tooltip"), "user"), + ("file", "str", _("Filemode for Downloads"), _("Tooltip"), "0644"), + ("change_group", "bool", _("Change group of running process"), _("Tooltip"), False), + ("folder", "str", _("Folder Permission mode"), _("Tooltip"), "0755"), + ("change_user", "bool", _("Change user of running process"), _("Tooltip"), False), + ], + True) + + config.addConfigSection("general", _("General"), _("Description"), _("Long description"), + [ + ("language", "en;de;fr;it;es;nl;sv;ru;pl;cs;sr;pt_BR", _("Language"), _("Tooltip"), "en"), + ("download_folder", "folder", _("Download Folder"), _("Tooltip"), "Downloads"), + ("checksum", "bool", _("Use Checksum"), _("Tooltip"), False), + ("folder_per_package", "bool", _("Create folder for each package"), _("Tooltip"), True), + ("debug_mode", "bool", _("Debug Mode"), _("Tooltip"), False), + ("min_free_space", "int", _("Min Free Space (MB)"), _("Tooltip"), 200), + ("renice", "int", _("CPU Priority"), _("Tooltip"), 0), + ], + True) + + config.addConfigSection("ssl", _("SSL"), _("Description"), _("Long description"), + [ + ("cert", "file", _("SSL Certificate"), _("Tooltip"), "ssl.crt"), + ("activated", "bool", _("Activated"), _("Tooltip"), False), + ("key", "file", _("SSL Key"), _("Tooltip"), "ssl.key"), + ], + True) + + config.addConfigSection("webinterface", _("Webinterface"), _("Description"), _("Long description"), + [ + ("template", "str", _("Template"), _("Tooltip"), "default"), + ("activated", "bool", _("Activated"), _("Tooltip"), True), + ("prefix", "str", _("Path Prefix"), _("Tooltip"), ""), + ("server", "builtin;threaded;fastcgi;lightweight", _("Server"), _("Tooltip"), "builtin"), + ("host", "ip", _("IP"), _("Tooltip"), "0.0.0.0"), + ("https", "bool", _("Use HTTPS"), _("Tooltip"), False), + ("port", "int", _("Port"), _("Tooltip"), 8001), + ], + True) + + config.addConfigSection("proxy", _("Proxy"), _("Description"), _("Long description"), + [ + ("username", "str", _("Username"), _("Tooltip"), ""), + ("proxy", "bool", _("Use Proxy"), _("Tooltip"), False), + ("address", "str", _("Address"), _("Tooltip"), "localhost"), + ("password", "password", _("Password"), _("Tooltip"), ""), + ("type", "http;socks4;socks5", _("Protocol"), _("Tooltip"), "http"), + ("port", "int", _("Port"), _("Tooltip"), 7070), + ], + True) + + config.addConfigSection("reconnect", _("Reconnect"), _("Description"), _("Long description"), + [ + ("endTime", "time", _("End"), _("Tooltip"), "0:00"), + ("activated", "bool", _("Use Reconnect"), _("Tooltip"), False), + ("method", "str", _("Method"), _("Tooltip"), "./reconnect.sh"), + ("startTime", "time", _("Start"), _("Tooltip"), "0:00"), + ], + True) + + config.addConfigSection("download", _("Download"), _("Description"), _("Long description"), + [ + ("max_downloads", "int", _("Max Parallel Downloads"), _("Tooltip"), 3), + ("limit_speed", "bool", _("Limit Download Speed"), _("Tooltip"), False), + ("interface", "str", _("Download interface to bind (ip or Name)"), _("Tooltip"), ""), + ("skip_existing", "bool", _("Skip already existing files"), _("Tooltip"), False), + ("max_speed", "int", _("Max Download Speed in kb/s"), _("Tooltip"), -1), + ("ipv6", "bool", _("Allow IPv6"), _("Tooltip"), False), + ("chunks", "int", _("Max connections for one download"), _("Tooltip"), 3), + ], + True) + + config.addConfigSection("downloadTime", _("Download Time"), _("Description"), _("Long description"), + [ + ("start", "time", _("Start"), _("Tooltip"), "0:00"), + ("end", "time", _("End"), _("Tooltip"), "0:00"), + ], + True) diff --git a/module/config/gui_default.xml b/module/config/gui_default.xml deleted file mode 100644 index 1faed776f..000000000 --- a/module/config/gui_default.xml +++ /dev/null @@ -1,13 +0,0 @@ -<?xml version="1.0" ?> -<root> - <connections> - <connection default="True" type="local" id="33965310e19b4a869112c43b39a16440"> - <name>Local</name> - </connection> - </connections> - <mainWindow> - <state></state> - <geometry></geometry> - </mainWindow> - <language>en</language> -</root> diff --git a/module/database/AccountDatabase.py b/module/database/AccountDatabase.py new file mode 100644 index 000000000..1602451fa --- /dev/null +++ b/module/database/AccountDatabase.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +from module.database import queue, async +from module.database import DatabaseBackend + +class AccountMethods: + + @queue + def loadAccounts(db): + db.c.execute('SELECT plugin, loginname, activated, password, options FROM accounts;') + return db.c.fetchall() + + @async + def saveAccounts(db, data): + db.c.executemany('INSERT INTO accounts(plugin, loginname, activated, password, options) VALUES(?,?,?,?,?)', data) + + @async + def removeAccount(db, plugin, loginname): + db.c.execute('DELETE FROM accounts WHERE plugin=? AND loginname=?', (plugin, loginname)) + +DatabaseBackend.registerSub(AccountMethods)
\ No newline at end of file diff --git a/module/database/DatabaseBackend.py b/module/database/DatabaseBackend.py index 9530390c3..32f75328c 100644 --- a/module/database/DatabaseBackend.py +++ b/module/database/DatabaseBackend.py @@ -25,62 +25,65 @@ from shutil import move from Queue import Queue from traceback import print_exc -from module.utils import chmod +from module.utils.fs import chmod try: from pysqlite2 import dbapi2 as sqlite3 except: import sqlite3 +DB = None DB_VERSION = 4 -class style(): - db = None - - @classmethod - def setDB(cls, db): - cls.db = db - - @classmethod - def inner(cls, f): - @staticmethod - def x(*args, **kwargs): - if cls.db: - return f(cls.db, *args, **kwargs) - return x - - @classmethod - def queue(cls, f): - @staticmethod - def x(*args, **kwargs): - if cls.db: - return cls.db.queue(f, *args, **kwargs) - return x - - @classmethod - def async(cls, f): - @staticmethod - def x(*args, **kwargs): - if cls.db: - return cls.db.async(f, *args, **kwargs) - return x +def set_DB(db): + global DB + DB = db + + +def queue(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return DB.queue(f, *args, **kwargs) + + return x + + +def async(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return DB.async(f, *args, **kwargs) + + return x + + +def inner(f): + @staticmethod + def x(*args, **kwargs): + if DB: + return f(DB, *args, **kwargs) + + return x + class DatabaseJob(): def __init__(self, f, *args, **kwargs): self.done = Event() - + self.f = f self.args = args self.kwargs = kwargs - + self.result = None self.exception = False -# import inspect -# self.frame = inspect.currentframe() + # import inspect + # self.frame = inspect.currentframe() def __repr__(self): from os.path import basename + frame = self.frame.f_back output = "" for i in range(5): @@ -104,46 +107,50 @@ class DatabaseJob(): self.exception = e finally: self.done.set() - + def wait(self): self.done.wait() + class DatabaseBackend(Thread): subs = [] + + DB_FILE = "pyload.db" + VERSION_FILE = "db.version" + def __init__(self, core): Thread.__init__(self) self.setDaemon(True) self.core = core self.jobs = Queue() - + self.setuplock = Event() - - style.setDB(self) - + + set_DB(self) + def setup(self): self.start() self.setuplock.wait() - + def run(self): """main loop, which executes commands""" convert = self._checkVersion() #returns None or current version - - self.conn = sqlite3.connect("files.db") - chmod("files.db", 0600) + + self.conn = sqlite3.connect(self.DB_FILE) + chmod(self.DB_FILE, 0600) self.c = self.conn.cursor() #compatibility - + if convert is not None: self._convertDB(convert) - + self._createTables() - self._migrateUser() self.conn.commit() - + self.setuplock.set() - + while True: j = self.jobs.get() if j == "quit": @@ -152,20 +159,23 @@ class DatabaseBackend(Thread): break j.processJob() - @style.queue + @queue def shutdown(self): self.conn.commit() self.jobs.put("quit") def _checkVersion(self): """ check db version and delete it if needed""" - if not exists("files.version"): - f = open("files.version", "wb") + if not exists(self.VERSION_FILE): + f = open(self.VERSION_FILE, "wb") f.write(str(DB_VERSION)) f.close() return - - f = open("files.version", "rb") + + if exists("files.db") and not exists(self.DB_FILE): + move("files.db", self.DB_FILE) + + f = open(self.VERSION_FILE, "rb") v = int(f.read().strip()) f.close() if v < DB_VERSION: @@ -174,13 +184,13 @@ class DatabaseBackend(Thread): self.manager.core.log.warning(_("Filedatabase was deleted due to incompatible version.")) except: print "Filedatabase was deleted due to incompatible version." - remove("files.version") - move("files.db", "files.backup.db") - f = open("files.version", "wb") + remove(self.VERSION_FILE) + move(self.DB_FILE, self.DB_FILE + ".backup") + f = open(self.VERSION_FILE, "wb") f.write(str(DB_VERSION)) f.close() return v - + def _convertDB(self, v): try: getattr(self, "_convertV%i" % v)() @@ -189,34 +199,28 @@ class DatabaseBackend(Thread): self.core.log.error(_("Filedatabase could NOT be converted.")) except: print "Filedatabase could NOT be converted." - + #--convert scripts start - - def _convertV2(self): - self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') - try: - self.manager.core.log.info(_("Database was converted from v2 to v3.")) - except: - print "Database was converted from v2 to v3." - self._convertV3() - - def _convertV3(self): - self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') - try: - self.manager.core.log.info(_("Database was converted from v3 to v4.")) - except: - print "Database was converted from v3 to v4." - + + def _convertV4(self): + pass + #--convert scripts end - + def _createTables(self): """create tables for database""" - self.c.execute('CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') - self.c.execute('CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "packages" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "folder" TEXT, "password" TEXT DEFAULT "", "site" TEXT DEFAULT "", "queue" INTEGER DEFAULT 0 NOT NULL, "packageorder" INTEGER DEFAULT 0 NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "links" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "url" TEXT NOT NULL, "name" TEXT, "size" INTEGER DEFAULT 0 NOT NULL, "status" INTEGER DEFAULT 3 NOT NULL, "plugin" TEXT DEFAULT "BasePlugin" NOT NULL, "error" TEXT DEFAULT "", "linkorder" INTEGER DEFAULT 0 NOT NULL, "package" INTEGER DEFAULT 0 NOT NULL, FOREIGN KEY(package) REFERENCES packages(id))') self.c.execute('CREATE INDEX IF NOT EXISTS "pIdIndex" ON links(package)') - self.c.execute('CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') - self.c.execute('CREATE TABLE IF NOT EXISTS "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "name" TEXT NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "storage" ("id" INTEGER PRIMARY KEY AUTOINCREMENT, "identifier" TEXT NOT NULL, "key" TEXT NOT NULL, "value" TEXT DEFAULT "")') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "users" ("name" TEXT PRIMARY KEY NOT NULL, "email" TEXT DEFAULT "" NOT NULL, "password" TEXT NOT NULL, "role" INTEGER DEFAULT 0 NOT NULL, "permission" INTEGER DEFAULT 0 NOT NULL, "template" TEXT DEFAULT "default" NOT NULL)') + self.c.execute( + 'CREATE TABLE IF NOT EXISTS "accounts" ("plugin" TEXT NOT NULL, "loginname" TEXT NOT NULL, "activated" INTEGER DEFAULT 1, "password" TEXT DEFAULT "", "options" TEXT DEFAULT "", PRIMARY KEY (plugin, loginname) ON CONFLICT REPLACE)') self.c.execute('CREATE VIEW IF NOT EXISTS "pstats" AS \ SELECT p.id AS id, SUM(l.size) AS sizetotal, COUNT(l.id) AS linkstotal, linksdone, sizedone\ @@ -234,7 +238,6 @@ class DatabaseBackend(Thread): fid = 0 self.c.execute('UPDATE SQLITE_SEQUENCE SET seq=? WHERE name=?', (fid, "links")) - self.c.execute('SELECT max(id) FROM packages') pid = self.c.fetchone()[0] if pid: @@ -246,60 +249,41 @@ class DatabaseBackend(Thread): self.c.execute('VACUUM') - def _migrateUser(self): - if exists("pyload.db"): - try: - self.core.log.info(_("Converting old Django DB")) - except: - print "Converting old Django DB" - conn = sqlite3.connect('pyload.db') - c = conn.cursor() - c.execute("SELECT username, password, email from auth_user WHERE is_superuser") - users = [] - for r in c: - pw = r[1].split("$") - users.append((r[0], pw[1] + pw[2], r[2])) - c.close() - conn.close() - - self.c.executemany("INSERT INTO users(name, password, email) VALUES (?, ?, ?)", users) - move("pyload.db", "pyload.old.db") - def createCursor(self): return self.conn.cursor() - - @style.async + + @async def commit(self): self.conn.commit() - @style.queue + @queue def syncSave(self): self.conn.commit() - - @style.async + + @async def rollback(self): self.conn.rollback() - + def async(self, f, *args, **kwargs): args = (self, ) + args job = DatabaseJob(f, *args, **kwargs) self.jobs.put(job) - + def queue(self, f, *args, **kwargs): args = (self, ) + args job = DatabaseJob(f, *args, **kwargs) self.jobs.put(job) job.wait() return job.result - + @classmethod def registerSub(cls, klass): cls.subs.append(klass) - + @classmethod def unregisterSub(cls, klass): cls.subs.remove(klass) - + def __getattr__(self, attr): for sub in DatabaseBackend.subs: if hasattr(sub, attr): @@ -308,45 +292,47 @@ class DatabaseBackend(Thread): if __name__ == "__main__": db = DatabaseBackend() db.setup() - + class Test(): - @style.queue + @queue def insert(db): c = db.createCursor() for i in range(1000): c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) - @style.async + + @async def insert2(db): c = db.createCursor() - for i in range(1000*1000): + for i in range(1000 * 1000): c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", ("foo", i, "bar")) - - @style.queue + + @queue def select(db): c = db.createCursor() for i in range(10): res = c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", ("foo", i)) print res.fetchone() - - @style.queue + + @queue def error(db): c = db.createCursor() print "a" c.execute("SELECT myerror FROM storage WHERE identifier=? AND key=?", ("foo", i)) print "e" - + db.registerSub(Test) from time import time + start = time() for i in range(100): db.insert() end = time() - print end-start - + print end - start + start = time() db.insert2() end = time() - print end-start - + print end - start + db.error() diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py index 357cd766d..eb76f468b 100644 --- a/module/database/FileDatabase.py +++ b/module/database/FileDatabase.py @@ -22,10 +22,9 @@ from threading import RLock from time import time from module.utils import formatSize, lock -from module.PullEvents import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent from module.PyPackage import PyPackage from module.PyFile import PyFile -from module.database import style, DatabaseBackend +from module.database import DatabaseBackend, queue, async, inner try: from pysqlite2 import dbapi2 as sqlite3 @@ -40,11 +39,12 @@ class FileHandler: def __init__(self, core): """Constructor""" self.core = core + self.evm = core.eventManager # translations self.statusMsg = [_("finished"), _("offline"), _("online"), _("queued"), _("skipped"), _("waiting"), _("temp. offline"), _("starting"), _("failed"), _("aborted"), _("decrypting"), _("custom"), _("downloading"), _("processing"), _("unknown")] - self.cache = {} #holds instances for files + self.cache = {} # holds instances for files self.packageCache = {} # same for packages #@TODO: purge the cache @@ -54,14 +54,12 @@ class FileHandler: #self.lock._Verbose__verbose = True self.filecount = -1 # if an invalid value is set get current value from db - self.queuecount = -1 #number of package to be loaded - self.unchanged = False #determines if any changes was made since last call + self.queuecount = -1 # number of package to be loaded self.db = self.core.db def change(func): def new(*args): - args[0].unchanged = False args[0].filecount = -1 args[0].queuecount = -1 args[0].jobCache = {} @@ -118,31 +116,23 @@ class FileHandler: @lock @change - def addLinks(self, urls, package): - """adds links""" - - self.core.hookManager.dispatchEvent("linksAdded", urls, package) - - data = self.core.pluginManager.parseUrls(urls) - + def addLinks(self, data, package): + """Add links, data = (plugin, url) tuple. Internal method you should use API.""" self.db.addLinks(data, package) - self.core.threadManager.createInfoThread(data, package) + self.evm.dispatchEvent("packageUpdated", package) - #@TODO change from reloadAll event to package update event - self.core.pullManager.addEvent(ReloadAllEvent("collector")) - #---------------------------------------------------------------------- @lock @change - def addPackage(self, name, folder, queue=0): + def addPackage(self, name, folder, queue=0, password=""): """adds a package, default to link collector""" - lastID = self.db.addPackage(name, folder, queue) - p = self.db.getPackage(lastID) - e = InsertEvent("pack", lastID, p.order, "collector" if not queue else "queue") - self.core.pullManager.addEvent(e) - return lastID + pid = self.db.addPackage(name, folder, queue, password) + p = self.db.getPackage(pid) + + self.evm.dispatchEvent("packageInserted", pid, p.queue, p.order) + return pid + - #---------------------------------------------------------------------- @lock @change def deletePackage(self, id): @@ -156,7 +146,6 @@ class FileHandler: oldorder = p.order queue = p.queue - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") pyfiles = self.cache.values() @@ -166,8 +155,7 @@ class FileHandler: pyfile.release() self.db.deletePackage(p) - self.core.pullManager.addEvent(e) - self.core.hookManager.dispatchEvent("packageDeleted", id) + self.evm.dispatchEvent("packageDeleted", id) if id in self.packageCache: del self.packageCache[id] @@ -178,7 +166,7 @@ class FileHandler: pack.order -= 1 pack.notifyChange() - #---------------------------------------------------------------------- + @lock @change def deleteLink(self, id): @@ -189,8 +177,6 @@ class FileHandler: return None pid = f.packageid - e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue") - oldorder = f.order if id in self.core.threadManager.processingIds(): @@ -201,11 +187,10 @@ class FileHandler: self.db.deleteLink(f) - self.core.pullManager.addEvent(e) + self.evm.dispatchEvent("linkDeleted", id, pid) p = self.getPackage(pid) - if not len(p.getChildren()): - p.delete() + p.deleteIfEmpty() pyfiles = self.cache.values() for pyfile in pyfiles: @@ -213,35 +198,26 @@ class FileHandler: pyfile.order -= 1 pyfile.notifyChange() - #---------------------------------------------------------------------- def releaseLink(self, id): """removes pyfile from cache""" if id in self.cache: del self.cache[id] - #---------------------------------------------------------------------- def releasePackage(self, id): """removes package from cache""" if id in self.packageCache: del self.packageCache[id] - #---------------------------------------------------------------------- def updateLink(self, pyfile): """updates link""" self.db.updateLink(pyfile) + self.evm.dispatchEvent("linkUpdated", pyfile.id, pyfile.packageid) - e = UpdateEvent("file", pyfile.id, "collector" if not pyfile.package().queue else "queue") - self.core.pullManager.addEvent(e) - - #---------------------------------------------------------------------- def updatePackage(self, pypack): """updates a package""" self.db.updatePackage(pypack) + self.evm.dispatchEvent("packageUpdated", pypack.id) - e = UpdateEvent("pack", pypack.id, "collector" if not pypack.queue else "queue") - self.core.pullManager.addEvent(e) - - #---------------------------------------------------------------------- def getPackage(self, id): """return package instance""" @@ -250,7 +226,6 @@ class FileHandler: else: return self.db.getPackage(id) - #---------------------------------------------------------------------- def getPackageData(self, id): """returns dict with package information""" pack = self.getPackage(id) @@ -274,7 +249,7 @@ class FileHandler: return pack - #---------------------------------------------------------------------- + def getFileData(self, id): """returns dict with file information""" if id in self.cache: @@ -282,7 +257,7 @@ class FileHandler: return self.db.getLinkData(id) - #---------------------------------------------------------------------- + def getFile(self, id): """returns pyfile instance""" if id in self.cache: @@ -290,7 +265,7 @@ class FileHandler: else: return self.db.getFile(id) - #---------------------------------------------------------------------- + @lock def getJob(self, occ): """get suitable job""" @@ -334,21 +309,6 @@ class FileHandler: #pyfile = self.getFile(self.jobCache[occ].pop()) return pyfile - @lock - def getDecryptJob(self): - """return job for decrypting""" - if "decrypt" in self.jobCache: - return None - - plugins = self.core.pluginManager.crypterPlugins.keys() + self.core.pluginManager.containerPlugins.keys() - plugins = str(tuple(plugins)) - - jobs = self.db.getPluginJob(plugins) - if jobs: - return self.getFile(jobs[0]) - else: - self.jobCache["decrypt"] = "empty" - return None def getFileCount(self): """returns number of files""" @@ -405,8 +365,7 @@ class FileHandler: if id in self.packageCache: self.packageCache[id].setFinished = False - e = UpdateEvent("pack", id, "collector" if not self.getPackage(id).queue else "queue") - self.core.pullManager.addEvent(e) + self.evm.dispatchEvent("packageUpdated", id) @lock @change @@ -420,9 +379,8 @@ class FileHandler: self.db.restartFile(id) + self.evm.dispatchEvent("linkUpdated", id) - e = UpdateEvent("file", id, "collector" if not self.getFile(id).package().queue else "queue") - self.core.pullManager.addEvent(e) @lock @change @@ -431,17 +389,10 @@ class FileHandler: p = self.db.getPackage(id) oldorder = p.order + p.queue = queue - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) - self.db.clearPackageOrder(p) - - p = self.db.getPackage(id) - - p.queue = queue self.db.updatePackage(p) - self.db.reorderPackage(p, -1, True) packs = self.packageCache.values() @@ -452,37 +403,34 @@ class FileHandler: self.db.commit() self.releasePackage(id) - p = self.getPackage(id) - - e = InsertEvent("pack", id, p.order, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) + + self.evm.dispatchEvent("packageDeleted", id) + self.evm.dispatchEvent("packageInserted", id, p.queue, p.order) @lock @change def reorderPackage(self, id, position): p = self.getPackage(id) - e = RemoveEvent("pack", id, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) self.db.reorderPackage(p, position) packs = self.packageCache.values() for pack in packs: if pack.queue != p.queue or pack.order < 0 or pack == p: continue if p.order > position: - if pack.order >= position and pack.order < p.order: + if position <= pack.order < p.order: pack.order += 1 pack.notifyChange() elif p.order < position: - if pack.order <= position and pack.order > p.order: + if position >= pack.order > p.order: pack.order -= 1 pack.notifyChange() p.order = position self.db.commit() - e = InsertEvent("pack", id, position, "collector" if not p.queue else "queue") - self.core.pullManager.addEvent(e) + self.evm.dispatchEvent("packageDeleted", id) + self.evm.dispatchEvent("packageInserted", id, p.queue, p.order) @lock @change @@ -490,20 +438,17 @@ class FileHandler: f = self.getFileData(id) f = f[id] - e = RemoveEvent("file", id, "collector" if not self.getPackage(f["package"]).queue else "queue") - self.core.pullManager.addEvent(e) - self.db.reorderLink(f, position) pyfiles = self.cache.values() for pyfile in pyfiles: if pyfile.packageid != f["package"] or pyfile.order < 0: continue if f["order"] > position: - if pyfile.order >= position and pyfile.order < f["order"]: + if position <= pyfile.order < f["order"]: pyfile.order += 1 pyfile.notifyChange() elif f["order"] < position: - if pyfile.order <= position and pyfile.order > f["order"]: + if position >= pyfile.order > f["order"]: pyfile.order -= 1 pyfile.notifyChange() @@ -512,15 +457,14 @@ class FileHandler: self.db.commit() - e = InsertEvent("file", id, position, "collector" if not self.getPackage(f["package"]).queue else "queue") - self.core.pullManager.addEvent(e) + self.evm.dispatchEvent("packageUpdated", f["package"]) + @change def updateFileInfo(self, data, pid): """ updates file info (name, size, status, url)""" ids = self.db.updateLinkInfo(data) - e = UpdateEvent("pack", pid, "collector" if not self.getPackage(pid).queue else "queue") - self.core.pullManager.addEvent(e) + self.evm.dispatchEvent("packageUpdated", pid) def checkPackageFinished(self, pyfile): """ checks if package is finished and calls hookmanager """ @@ -574,25 +518,25 @@ class FileHandler: self.db.restartFailed() class FileMethods(): - @style.queue + @queue def filecount(self, queue): """returns number of files in queue""" self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=?", (queue, )) return self.c.fetchone()[0] - @style.queue + @queue def queuecount(self, queue): """ number of files in queue not finished yet""" self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status NOT IN (0,4)", (queue, )) return self.c.fetchone()[0] - @style.queue + @queue def processcount(self, queue, fid): """ number of files which have to be proccessed """ self.c.execute("SELECT COUNT(*) FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=? AND l.status IN (2,3,5,7,12) AND l.id != ?", (queue, str(fid))) return self.c.fetchone()[0] - @style.inner + @inner def _nextPackageOrder(self, queue=0): self.c.execute('SELECT MAX(packageorder) FROM packages WHERE queue=?', (queue,)) max = self.c.fetchone()[0] @@ -601,7 +545,7 @@ class FileMethods(): else: return 0 - @style.inner + @inner def _nextFileOrder(self, package): self.c.execute('SELECT MAX(linkorder) FROM links WHERE package=?', (package,)) max = self.c.fetchone()[0] @@ -610,13 +554,13 @@ class FileMethods(): else: return 0 - @style.queue + @queue def addLink(self, url, name, plugin, package): order = self._nextFileOrder(package) self.c.execute('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', (url, name, plugin, package, order)) return self.c.lastrowid - @style.queue + @queue def addLinks(self, links, package): """ links is a list of tupels (url,plugin)""" order = self._nextFileOrder(package) @@ -624,27 +568,27 @@ class FileMethods(): links = [(x[0], x[0], x[1], package, o) for x, o in zip(links, orders)] self.c.executemany('INSERT INTO links(url, name, plugin, package, linkorder) VALUES(?,?,?,?,?)', links) - @style.queue - def addPackage(self, name, folder, queue): + @queue + def addPackage(self, name, folder, queue, password): order = self._nextPackageOrder(queue) - self.c.execute('INSERT INTO packages(name, folder, queue, packageorder) VALUES(?,?,?,?)', (name, folder, queue, order)) + self.c.execute('INSERT INTO packages(name, folder, queue, packageorder, password) VALUES(?,?,?,?,?)', (name, folder, queue, order, password)) return self.c.lastrowid - @style.queue + @queue def deletePackage(self, p): self.c.execute('DELETE FROM links WHERE package=?', (str(p.id),)) self.c.execute('DELETE FROM packages WHERE id=?', (str(p.id),)) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=?', (p.order, p.queue)) - @style.queue + @queue def deleteLink(self, f): self.c.execute('DELETE FROM links WHERE id=?', (str(f.id),)) self.c.execute('UPDATE links SET linkorder=linkorder-1 WHERE linkorder > ? AND package=?', (f.order, str(f.packageid))) - @style.queue + @queue def getAllLinks(self, q): """return information about all links in queue q @@ -677,7 +621,7 @@ class FileMethods(): return data - @style.queue + @queue def getAllPackages(self, q): """return information about packages in queue q (only useful in get all data) @@ -692,7 +636,7 @@ class FileMethods(): } """ self.c.execute('SELECT p.id, p.name, p.folder, p.site, p.password, p.queue, p.packageorder, s.sizetotal, s.sizedone, s.linksdone, s.linkstotal \ - FROM packages p JOIN pstats s ON p.id = s.id \ + FROM packages p LEFT OUTER JOIN pstats s ON p.id = s.id \ WHERE p.queue=? ORDER BY p.packageorder', str(q)) data = {} @@ -705,16 +649,16 @@ class FileMethods(): 'password': r[4], 'queue': r[5], 'order': r[6], - 'sizetotal': int(r[7]), - 'sizedone': r[8] if r[8] else 0, #these can be None + 'sizetotal': int(r[7]) if r[7] else 0, + 'sizedone': int(r[8]) if r[8] else 0, #these can be None 'linksdone': r[9] if r[9] else 0, - 'linkstotal': r[10], + 'linkstotal': r[10] if r[10] else 0, 'links': {} } return data - @style.queue + @queue def getLinkData(self, id): """get link information as dict""" self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE id=?', (str(id), )) @@ -738,7 +682,7 @@ class FileMethods(): return data - @style.queue + @queue def getPackageData(self, id): """get data about links for a package""" self.c.execute('SELECT id,url,name,size,status,error,plugin,package,linkorder FROM links WHERE package=? ORDER BY linkorder', (str(id), )) @@ -762,15 +706,15 @@ class FileMethods(): return data - @style.async + @async def updateLink(self, f): self.c.execute('UPDATE links SET url=?,name=?,size=?,status=?,error=?,package=? WHERE id=?', (f.url, f.name, f.size, f.status, f.error, str(f.packageid), str(f.id))) - @style.queue + @queue def updatePackage(self, p): self.c.execute('UPDATE packages SET name=?,folder=?,site=?,password=?,queue=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, str(p.id))) - @style.queue + @queue def updateLinkInfo(self, data): """ data is list of tupels (name, size, status, url) """ self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=? AND status IN (1,2,3,14)', data) @@ -780,7 +724,7 @@ class FileMethods(): ids.append(int(r[0])) return ids - @style.queue + @queue def reorderPackage(self, p, position, noMove=False): if position == -1: position = self._nextPackageOrder(p.queue) @@ -792,7 +736,7 @@ class FileMethods(): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (position, str(p.id))) - @style.queue + @queue def reorderLink(self, f, position): """ reorder link with f as dict for pyfile """ if f["order"] > position: @@ -803,20 +747,20 @@ class FileMethods(): self.c.execute('UPDATE links SET linkorder=? WHERE id=?', (position, f["id"])) - @style.queue + @queue def clearPackageOrder(self, p): self.c.execute('UPDATE packages SET packageorder=? WHERE id=?', (-1, str(p.id))) self.c.execute('UPDATE packages SET packageorder=packageorder-1 WHERE packageorder > ? AND queue=? AND id != ?', (p.order, p.queue, str(p.id))) - @style.async + @async def restartFile(self, id): self.c.execute('UPDATE links SET status=3,error="" WHERE id=?', (str(id),)) - @style.async + @async def restartPackage(self, id): self.c.execute('UPDATE links SET status=3 WHERE package=?', (str(id),)) - @style.queue + @queue def getPackage(self, id): """return package instance from id""" self.c.execute("SELECT name,folder,site,password,queue,packageorder FROM packages WHERE id=?", (str(id), )) @@ -824,8 +768,8 @@ class FileMethods(): if not r: return None return PyPackage(self.manager, id, * r) - #---------------------------------------------------------------------- - @style.queue + + @queue def getFile(self, id): """return link instance from id""" self.c.execute("SELECT url, name, size, status, error, plugin, package, linkorder FROM links WHERE id=?", (str(id), )) @@ -834,58 +778,44 @@ class FileMethods(): return PyFile(self.manager, id, * r) - @style.queue + @queue def getJob(self, occ): """return pyfile ids, which are suitable for download and dont use a occupied plugin""" - - #@TODO improve this hardcoded method - pre = "('DLC', 'LinkList', 'SerienjunkiesOrg', 'CCF', 'RSDF')" #plugins which are processed in collector - cmd = "(" for i, item in enumerate(occ): if i: cmd += ", " cmd += "'%s'" % item - - cmd += ")" - - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE ((p.queue=1 AND l.plugin NOT IN %s) OR l.plugin IN %s) AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % (cmd, pre) - - self.c.execute(cmd) # very bad! - return [x[0] for x in self.c] - - @style.queue - def getPluginJob(self, plugins): - """returns pyfile ids with suited plugins""" - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.plugin IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % plugins + cmd += ")" + cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE p.queue=1 AND l.plugin NOT IN %s AND l.status IN (2,3,14) ORDER BY p.packageorder ASC, l.linkorder ASC LIMIT 5" % cmd self.c.execute(cmd) # very bad! return [x[0] for x in self.c] - @style.queue + @queue def getUnfinished(self, pid): """return list of max length 3 ids with pyfiles in package not finished or processed""" self.c.execute("SELECT id FROM links WHERE package=? AND status NOT IN (0, 4, 13) LIMIT 3", (str(pid),)) return [r[0] for r in self.c] - @style.queue + @queue def deleteFinished(self): self.c.execute("DELETE FROM links WHERE status IN (0,4)") self.c.execute("DELETE FROM packages WHERE NOT EXISTS(SELECT 1 FROM links WHERE packages.id=links.package)") - @style.queue + @queue def restartFailed(self): self.c.execute("UPDATE links SET status=3,error='' WHERE status IN (8, 9)") - @style.queue + @queue def findDuplicates(self, id, folder, filename): """ checks if filename exists with different id and same package """ self.c.execute("SELECT l.plugin FROM links as l INNER JOIN packages as p ON l.package=p.id AND p.folder=? WHERE l.id!=? AND l.status=0 AND l.name=?", (folder, id, filename)) return self.c.fetchone() - @style.queue + @queue def purgeLinks(self): self.c.execute("DELETE FROM links;") self.c.execute("DELETE FROM packages;") diff --git a/module/database/StorageDatabase.py b/module/database/StorageDatabase.py index 3ed29625f..ffaf51763 100644 --- a/module/database/StorageDatabase.py +++ b/module/database/StorageDatabase.py @@ -16,11 +16,10 @@ @author: mkaay """ -from module.database import style -from module.database import DatabaseBackend +from module.database import DatabaseBackend, queue class StorageMethods(): - @style.queue + @queue def setStorage(db, identifier, key, value): db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) if db.c.fetchone() is not None: @@ -28,7 +27,7 @@ class StorageMethods(): else: db.c.execute("INSERT INTO storage (identifier, key, value) VALUES (?, ?, ?)", (identifier, key, value)) - @style.queue + @queue def getStorage(db, identifier, key=None): if key is not None: db.c.execute("SELECT value FROM storage WHERE identifier=? AND key=?", (identifier, key)) @@ -42,7 +41,7 @@ class StorageMethods(): d[row[0]] = row[1] return d - @style.queue + @queue def delStorage(db, identifier, key): db.c.execute("DELETE FROM storage WHERE identifier=? AND key=?", (identifier, key)) diff --git a/module/database/UserDatabase.py b/module/database/UserDatabase.py index 0c781057d..43fd93df3 100644 --- a/module/database/UserDatabase.py +++ b/module/database/UserDatabase.py @@ -19,14 +19,13 @@ from hashlib import sha1 import random -from DatabaseBackend import DatabaseBackend -from DatabaseBackend import style +from DatabaseBackend import DatabaseBackend, queue, async class UserMethods(): - @style.queue + @queue def checkAuth(db, user, password): c = db.c - c.execute('SELECT id, name, password, role, permission, template, email FROM "users" WHERE name=?', (user, )) + c.execute('SELECT rowid, name, password, role, permission, template, email FROM "users" WHERE name=?', (user, )) r = c.fetchone() if not r: return {} @@ -40,7 +39,7 @@ class UserMethods(): else: return {} - @style.queue + @queue def addUser(db, user, password): salt = reduce(lambda x, y: x + y, [str(random.randint(0, 9)) for i in range(0, 5)]) h = sha1(salt + password) @@ -54,9 +53,9 @@ class UserMethods(): c.execute('INSERT INTO users (name, password) VALUES (?, ?)', (user, password)) - @style.queue + @queue def changePassword(db, user, oldpw, newpw): - db.c.execute('SELECT id, name, password FROM users WHERE name=?', (user, )) + db.c.execute('SELECT rowid, name, password FROM users WHERE name=?', (user, )) r = db.c.fetchone() if not r: return False @@ -75,16 +74,16 @@ class UserMethods(): return False - @style.async + @async def setPermission(db, user, perms): db.c.execute("UPDATE users SET permission=? WHERE name=?", (perms, user)) - @style.async + @async def setRole(db, user, role): db.c.execute("UPDATE users SET role=? WHERE name=?", (role, user)) - @style.queue + @queue def listUsers(db): db.c.execute('SELECT name FROM users') users = [] @@ -92,7 +91,7 @@ class UserMethods(): users.append(row[0]) return users - @style.queue + @queue def getAllUserData(db): db.c.execute("SELECT name, permission, role, template, email FROM users") user = {} @@ -101,7 +100,7 @@ class UserMethods(): return user - @style.queue + @queue def removeUser(db, user): db.c.execute('DELETE FROM users WHERE name=?', (user, )) diff --git a/module/database/__init__.py b/module/database/__init__.py index 545789c0c..39848ac58 100644 --- a/module/database/__init__.py +++ b/module/database/__init__.py @@ -1,6 +1,6 @@ -from DatabaseBackend import DatabaseBackend -from DatabaseBackend import style +from DatabaseBackend import DatabaseBackend, queue, async, inner from FileDatabase import FileHandler from UserDatabase import UserMethods -from StorageDatabase import StorageMethods
\ No newline at end of file +from StorageDatabase import StorageMethods +from AccountDatabase import AccountMethods
\ No newline at end of file diff --git a/module/gui/AccountEdit.py b/module/gui/AccountEdit.py deleted file mode 100644 index b22cfc49f..000000000 --- a/module/gui/AccountEdit.py +++ /dev/null @@ -1,104 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from PyQt4.QtCore import * -from PyQt4.QtGui import * - -from os.path import join - -class AccountEdit(QWidget): - """ - account editor widget - """ - - def __init__(self): - QMainWindow.__init__(self) - - self.setWindowTitle(_("Edit account")) - self.setWindowIcon(QIcon(join(pypath, "icons","logo.png"))) - - self.setLayout(QGridLayout()) - l = self.layout() - - typeLabel = QLabel(_("Type")) - loginLabel = QLabel(_("Login")) - passwordLabel = QLabel(_("New password")) - changePw = QCheckBox() - changePw.setChecked(False) - self.changePw = changePw - password = QLineEdit() - password.setEnabled(False) - password.setEchoMode(QLineEdit.Password) - self.password = password - login = QLineEdit() - self.login = login - acctype = QComboBox() - self.acctype = acctype - - save = QPushButton(_("Save")) - - self.connect(changePw, SIGNAL("toggled(bool)"), password, SLOT("setEnabled(bool)")) - - l.addWidget(save, 3, 0, 1, 3) - l.addWidget(acctype, 0, 1, 1, 2) - l.addWidget(login, 1, 1, 1, 2) - l.addWidget(password, 2, 2) - l.addWidget(changePw, 2, 1) - l.addWidget(passwordLabel, 2, 0) - l.addWidget(loginLabel, 1, 0) - l.addWidget(typeLabel, 0, 0) - - self.connect(save, SIGNAL("clicked()"), self.slotSave) - - def slotSave(self): - """ - save entered data - """ - data = {"login": str(self.login.text()), "acctype": str(self.acctype.currentText()), "password": False} - if self.changePw.isChecked(): - data["password"] = str(self.password.text()) - self.emit(SIGNAL("done"), data) - - @staticmethod - def newAccount(types): - """ - create empty editor instance - """ - w = AccountEdit() - w.setWindowTitle(_("Create account")) - - w.changePw.setChecked(True) - w.password.setEnabled(True) - - w.acctype.addItems(types) - - return w - - @staticmethod - def editAccount(types, base): - """ - create editor instance with given data - """ - w = AccountEdit() - - w.acctype.addItems(types) - w.acctype.setCurrentIndex(types.index(base["type"])) - - w.login.setText(base["login"]) - - return w diff --git a/module/gui/Accounts.py b/module/gui/Accounts.py deleted file mode 100644 index 8db04dfa9..000000000 --- a/module/gui/Accounts.py +++ /dev/null @@ -1,213 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from PyQt4.QtCore import * -from PyQt4.QtGui import * - -from time import strftime, gmtime - -class AccountModel(QAbstractItemModel): - """ - model for account view - """ - - def __init__(self, view, connector): - QAbstractItemModel.__init__(self) - self.connector = connector - self.view = view - self._data = [] - self.cols = 4 - self.mutex = QMutex() - - def reloadData(self, force=False): - """ - reload account list - """ - accounts = self.connector.proxy.getAccounts(False) - - if self._data == accounts: - return - - if len(self._data) > 0: - self.beginRemoveRows(QModelIndex(), 0, len(self._data)-1) - self._data = [] - self.endRemoveRows() - - if len(accounts) > 0: - self.beginInsertRows(QModelIndex(), 0, len(accounts)-1) - self._data = accounts - self.endInsertRows() - - def toData(self, index): - """ - return index pointer - """ - return index.internalPointer() - - def data(self, index, role=Qt.DisplayRole): - """ - return cell data - """ - if not index.isValid(): - return QVariant() - if role == Qt.DisplayRole: - if index.column() == 0: - return QVariant(self.toData(index).type) - elif index.column() == 1: - return QVariant(self.toData(index).login) - elif index.column() == 2: - if not self.toData(index).valid: - return QVariant(_("not valid")) - if not self.toData(index).validuntil: - return QVariant(_("n/a")) - until = int(self.toData(index).validuntil) - if until > 0: - fmtime = strftime(_("%a, %d %b %Y %H:%M"), gmtime(until)) - return QVariant(fmtime) - else: - return QVariant(_("unlimited")) - #elif role == Qt.EditRole: - # if index.column() == 0: - # return QVariant(index.internalPointer().data["name"]) - return QVariant() - - def index(self, row, column, parent=QModelIndex()): - """ - create index with data pointer - """ - if parent == QModelIndex() and len(self._data) > row: - pointer = self._data[row] - index = self.createIndex(row, column, pointer) - elif parent.isValid(): - pointer = parent.internalPointer().children[row] - index = self.createIndex(row, column, pointer) - else: - index = QModelIndex() - return index - - def parent(self, index): - """ - no parents, everything on top level - """ - return QModelIndex() - - def rowCount(self, parent=QModelIndex()): - """ - account count - """ - if parent == QModelIndex(): - return len(self._data) - return 0 - - def columnCount(self, parent=QModelIndex()): - return self.cols - - def hasChildren(self, parent=QModelIndex()): - """ - everything on top level - """ - if parent == QModelIndex(): - return True - else: - return False - - def canFetchMore(self, parent): - return False - - def headerData(self, section, orientation, role=Qt.DisplayRole): - """ - returns column heading - """ - if orientation == Qt.Horizontal and role == Qt.DisplayRole: - if section == 0: - return QVariant(_("Type")) - elif section == 1: - return QVariant(_("Login")) - elif section == 2: - return QVariant(_("Valid until")) - elif section == 3: - return QVariant(_("Traffic left")) - return QVariant() - - def flags(self, index): - """ - cell flags - """ - return Qt.ItemIsSelectable | Qt.ItemIsEnabled - -class AccountView(QTreeView): - """ - view component for accounts - """ - - def __init__(self, connector): - QTreeView.__init__(self) - self.setModel(AccountModel(self, connector)) - - self.setColumnWidth(0, 150) - self.setColumnWidth(1, 150) - self.setColumnWidth(2, 150) - self.setColumnWidth(3, 150) - - self.setEditTriggers(QAbstractItemView.NoEditTriggers) - - self.delegate = AccountDelegate(self, self.model()) - self.setItemDelegateForColumn(3, self.delegate) - -class AccountDelegate(QItemDelegate): - """ - used to display a progressbar for the traffic in the traffic cell - """ - - def __init__(self, parent, model): - QItemDelegate.__init__(self, parent) - self.model = model - - def paint(self, painter, option, index): - """ - paint the progressbar - """ - if not index.isValid(): - return - if index.column() == 3: - data = self.model.toData(index) - opts = QStyleOptionProgressBarV2() - opts.minimum = 0 - if data.trafficleft: - if data.trafficleft == -1 or data.trafficleft is None: - opts.maximum = opts.progress = 1 - else: - opts.maximum = opts.progress = data.trafficleft - if data.maxtraffic: - opts.maximum = data.maxtraffic - - opts.rect = option.rect - opts.rect.setRight(option.rect.right()-1) - opts.rect.setHeight(option.rect.height()-1) - opts.textVisible = True - opts.textAlignment = Qt.AlignCenter - if data.trafficleft and data.trafficleft == -1: - opts.text = QString(_("unlimited")) - elif data.trafficleft is None: - opts.text = QString(_("n/a")) - else: - opts.text = QString.number(round(float(opts.progress)/1024/1024, 2)) + " GB" - QApplication.style().drawControl(QStyle.CE_ProgressBar, opts, painter) - return - QItemDelegate.paint(self, painter, option, index) - diff --git a/module/gui/CNLServer.py b/module/gui/CNLServer.py deleted file mode 100644 index 5ecac8277..000000000 --- a/module/gui/CNLServer.py +++ /dev/null @@ -1,226 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" -import re -from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler -from threading import Thread -from cgi import FieldStorage -from os.path import abspath, dirname, join -from urllib import unquote -from base64 import standard_b64decode -from binascii import unhexlify - -try: - from Crypto.Cipher import AES -except: - pass - -try: - from module.common import JsEngine -except ImportError: - import sys - sys.path.append(join(abspath(dirname(__file__)), "..", "..")) - from module.common import JsEngine - -js = JsEngine.JsEngine() -core = None - -class CNLServer(Thread): - def __init__(self): - Thread.__init__(self) - self.setDaemon(True) - - self.stop = False - self.stopped = False - - def run(self): - server_address = ('127.0.0.1', 9666) - try: - httpd = HTTPServer(server_address, CNLHandler) - except: - self.stopped = True - return - - self.stopped = False - while self.keep_running(): - httpd.handle_request() - self.stopped = True - - def keep_running(self): - return not self.stop - - -class CNLHandler(BaseHTTPRequestHandler): - - #def log_message(self, *args): - # pass - - def add_package(self, name, urls, queue=0): - print "name", name - print "urls", urls - print "queue", queue - - def get_post(self, name, default=""): - if name in self.post: - return self.post[name] - else: - return default - - def start_response(self, string): - - self.send_response(200) - - self.send_header("Content-Length", len(string)) - self.send_header("Content-Language", "de") - self.send_header("Vary", "Accept-Language, Cookie") - self.send_header("Cache-Control", "no-cache, must-revalidate") - self.send_header("Content-type", "text/html") - self.end_headers() - - def do_GET(self): - path = self.path.strip("/").lower() - #self.wfile.write(path+"\n") - - self.map = [ (r"add$", self.add), - (r"addcrypted$", self.addcrypted), - (r"addcrypted2$", self.addcrypted2), - (r"flashgot", self.flashgot), - (r"crossdomain\.xml", self.crossdomain), - (r"checkSupportForUrl", self.checksupport), - (r"jdcheck.js", self.jdcheck), - (r"", self.flash) ] - - func = None - for r, f in self.map: - if re.match(r"(flash(got)?/?)?"+r, path): - func = f - break - - if func: - try: - resp = func() - if not resp: resp = "success" - resp += "\r\n" - self.start_response(resp) - self.wfile.write(resp) - except Exception,e : - self.send_error(500, str(e)) - else: - self.send_error(404, "Not Found") - - def do_POST(self): - form = FieldStorage( - fp=self.rfile, - headers=self.headers, - environ={'REQUEST_METHOD':'POST', - 'CONTENT_TYPE':self.headers['Content-Type'], - }) - - self.post = {} - for name in form.keys(): - self.post[name] = form[name].value - - return self.do_GET() - - def flash(self): - return "JDownloader" - - def add(self): - package = self.get_post('referer', 'ClickAndLoad Package') - urls = filter(lambda x: x != "", self.get_post('urls').split("\n")) - - self.add_package(package, urls, 0) - - def addcrypted(self): - package = self.get_post('referer', 'ClickAndLoad Package') - dlc = self.get_post('crypted').replace(" ", "+") - - core.upload_container(package, dlc) - - def addcrypted2(self): - package = self.get_post("source", "ClickAndLoad Package") - crypted = self.get_post("crypted") - jk = self.get_post("jk") - - crypted = standard_b64decode(unquote(crypted.replace(" ", "+"))) - jk = "%s f()" % jk - jk = js.eval(jk) - Key = unhexlify(jk) - IV = Key - - obj = AES.new(Key, AES.MODE_CBC, IV) - result = obj.decrypt(crypted).replace("\x00", "").replace("\r","").split("\n") - - result = filter(lambda x: x != "", result) - - self.add_package(package, result, 0) - - - def flashgot(self): - autostart = int(self.get_post('autostart', 0)) - package = self.get_post('package', "FlashGot") - urls = filter(lambda x: x != "", self.get_post('urls').split("\n")) - - self.add_package(package, urls, autostart) - - def crossdomain(self): - rep = "<?xml version=\"1.0\"?>\n" - rep += "<!DOCTYPE cross-domain-policy SYSTEM \"http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd\">\n" - rep += "<cross-domain-policy>\n" - rep += "<allow-access-from domain=\"*\" />\n" - rep += "</cross-domain-policy>" - return rep - - def checksupport(self): - pass - - def jdcheck(self): - rep = "jdownloader=true;\n" - rep += "var version='10629';\n" - return rep - - -if __name__ == "__main__": - import xmlrpclib - from module import InitHomeDir - from module.ConfigParser import ConfigParser - - config = ConfigParser() - - ssl = "" - if config.get("ssl", "activated"): - ssl = "s" - - server_url = "http%s://%s:%s@%s:%s/" % ( - ssl, - config.username, - config.password, - config.get("remote", "listenaddr"), - config.get("remote", "port") - ) - - core = xmlrpclib.ServerProxy(server_url, allow_none=True) - - s = CNLServer() - s.start() - while not s.stopped: - try: - s.join(1) - except KeyboardInterrupt: - s.stop = True - s.stopped = True - print "quiting.." diff --git a/module/gui/CaptchaDock.py b/module/gui/CaptchaDock.py deleted file mode 100644 index b88cb53ca..000000000 --- a/module/gui/CaptchaDock.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from PyQt4.QtCore import * -from PyQt4.QtGui import * - -class CaptchaDock(QDockWidget): - """ - dock widget for captcha input - """ - - def __init__(self): - QDockWidget.__init__(self, _("Captcha")) - self.setObjectName("Captcha Dock") - self.widget = CaptchaDockWidget(self) - self.setWidget(self.widget) - self.setAllowedAreas(Qt.BottomDockWidgetArea) - self.setFeatures(QDockWidget.NoDockWidgetFeatures) - self.hide() - self.processing = False - self.currentID = None - self.connect(self, SIGNAL("setTask"), self.setTask) - - def isFree(self): - return not self.processing - - def setTask(self, tid, img, imgType): - self.processing = True - data = QByteArray(img) - self.currentID = tid - self.widget.emit(SIGNAL("setImage"), data) - self.widget.input.setText("") - self.show() - -class CaptchaDockWidget(QWidget): - """ - widget for the input widgets - """ - - def __init__(self, dock): - QWidget.__init__(self) - self.dock = dock - self.setLayout(QHBoxLayout()) - layout = self.layout() - - imgLabel = QLabel() - captchaInput = QLineEdit() - okayButton = QPushButton(_("OK")) - cancelButton = QPushButton(_("Cancel")) - - layout.addStretch() - layout.addWidget(imgLabel) - layout.addWidget(captchaInput) - layout.addWidget(okayButton) - layout.addWidget(cancelButton) - layout.addStretch() - - self.input = captchaInput - - self.connect(okayButton, SIGNAL("clicked()"), self.slotSubmit) - self.connect(captchaInput, SIGNAL("returnPressed()"), self.slotSubmit) - self.connect(self, SIGNAL("setImage"), self.setImg) - self.connect(self, SIGNAL("setPixmap(const QPixmap &)"), imgLabel, SLOT("setPixmap(const QPixmap &)")) - - def setImg(self, data): - pixmap = QPixmap() - pixmap.loadFromData(data) - self.emit(SIGNAL("setPixmap(const QPixmap &)"), pixmap) - self.input.setFocus(Qt.OtherFocusReason) - - def slotSubmit(self): - text = self.input.text() - tid = self.dock.currentID - self.dock.currentID = None - self.dock.emit(SIGNAL("done"), tid, str(text)) - self.dock.hide() - self.dock.processing = False - diff --git a/module/gui/Collector.py b/module/gui/Collector.py deleted file mode 100644 index 3ec4262f1..000000000 --- a/module/gui/Collector.py +++ /dev/null @@ -1,407 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from PyQt4.QtCore import * -from PyQt4.QtGui import * - -from module.PyFile import statusMap -from module.utils import formatSize - -from module.remote.thriftbackend.ThriftClient import Destination, FileDoesNotExists, ElementType - -statusMapReverse = dict((v,k) for k, v in statusMap.iteritems()) - -translatedStatusMap = {} # -> CollectorModel.__init__ - -class CollectorModel(QAbstractItemModel): - """ - model for the collector view - """ - - def __init__(self, view, connector): - QAbstractItemModel.__init__(self) - self.connector = connector - self.view = view - self._data = [] - self.cols = 4 - self.interval = 1 - self.mutex = QMutex() - - global translatedStatusMap # workaround because i18n is not running at import time - translatedStatusMap = { - "finished": _("finished"), - "offline": _("offline"), - "online": _("online"), - "queued": _("queued"), - "skipped": _("skipped"), - "waiting": _("waiting"), - "temp. offline": _("temp. offline"), - "starting": _("starting"), - "failed": _("failed"), - "aborted": _("aborted"), - "decrypting": _("decrypting"), - "custom": _("custom"), - "downloading": _("downloading"), - "processing": _("processing") - } - - def translateStatus(self, string): - """ - used to convert to locale specific status - """ - return translatedStatusMap[string] - - def addEvent(self, event): - """ - called from main loop, pass events to the correct methods - """ - QMutexLocker(self.mutex) - if event.eventname == "reload": - self.fullReload() - elif event.eventname == "remove": - self.removeEvent(event) - elif event.eventname == "insert": - self.insertEvent(event) - elif event.eventname == "update": - self.updateEvent(event) - - def fullReload(self): - """ - reload whole model, used at startup to load initial data - """ - self._data = [] - order = self.connector.getPackageOrder(Destination.Collector) - self.beginInsertRows(QModelIndex(), 0, len(order.values())) - for position, pid in order.iteritems(): - pack = self.connector.getPackageData(pid) - package = Package(pack) - self._data.append(package) - self._data = sorted(self._data, key=lambda p: p.data["order"]) - self.endInsertRows() - - def removeEvent(self, event): - """ - remove an element from model - """ - if event.type == ElementType.File: - for p, package in enumerate(self._data): - for k, child in enumerate(package.children): - if child.id == event.id: - self.beginRemoveRows(self.index(p, 0), k, k) - del package.children[k] - self.endRemoveRows() - break - else: - for k, package in enumerate(self._data): - if package.id == event.id: - self.beginRemoveRows(QModelIndex(), k, k) - del self._data[k] - self.endRemoveRows() - break - - def insertEvent(self, event): - """ - inserts a new element in the model - """ - if event.type == ElementType.File: - try: - info = self.connector.getFileData(event.id) - except FileDoesNotExists: - return - - for k, package in enumerate(self._data): - if package.id == info.package: - if package.getChild(info.fid): - self.updateEvent(event) - break - self.beginInsertRows(self.index(k, 0), info.order, info.order) - package.addChild(info) - self.endInsertRows() - break - else: - data = self.connector.getPackageData(event.id) - package = Package(data) - self.beginInsertRows(QModelIndex(), data.order, data.order) - self._data.insert(data.order, package) - self.endInsertRows() - - def updateEvent(self, event): - """ - update an element in the model - """ - if event.type == ElementType.File: - try: - info = self.connector.proxy.getFileData(event.id) - except FileDoesNotExists: - return - for p, package in enumerate(self._data): - if package.id == info.packageID: - for k, child in enumerate(package.children): - if child.id == event.id: - child.update(info) - if not info.status == 12: - child.data["downloading"] = None - self.emit(SIGNAL("dataChanged(const QModelIndex &, const QModelIndex &)"), self.index(k, 0, self.index(p, 0)), self.index(k, self.cols, self.index(p, self.cols))) - break - else: - data = self.connector.getPackageData(event.id) - if not data: - return - for p, package in enumerate(self._data): - if package.id == event.id: - package.update(data) - self.emit(SIGNAL("dataChanged(const QModelIndex &, const QModelIndex &)"), self.index(p, 0), self.index(p, self.cols)) - break - - def data(self, index, role=Qt.DisplayRole): - """ - return cell data - """ - if not index.isValid(): - return QVariant() - if role == Qt.DisplayRole: - if index.column() == 0: - return QVariant(index.internalPointer().data["name"]) - elif index.column() == 1: - item = index.internalPointer() - plugins = [] - if isinstance(item, Package): - for child in item.children: - if not child.data["plugin"] in plugins: - plugins.append(child.data["plugin"]) - else: - plugins.append(item.data["plugin"]) - return QVariant(", ".join(plugins)) - elif index.column() == 2: - item = index.internalPointer() - status = 0 - if isinstance(item, Package): - for child in item.children: - if child.data["status"] > status: - status = child.data["status"] - else: - status = item.data["status"] - return QVariant(self.translateStatus(statusMapReverse[status])) - elif index.column() == 3: - item = index.internalPointer() - if isinstance(item, Link): - return QVariant(formatSize(item.data["size"])) - else: - ms = 0 - for c in item.children: - ms += c.data["size"] - return QVariant(formatSize(ms)) - elif role == Qt.EditRole: - if index.column() == 0: - return QVariant(index.internalPointer().data["name"]) - return QVariant() - - def index(self, row, column, parent=QModelIndex()): - """ - creates a cell index with pointer to the data - """ - if parent == QModelIndex() and len(self._data) > row: - pointer = self._data[row] - index = self.createIndex(row, column, pointer) - elif parent.isValid(): - try: - pointer = parent.internalPointer().children[row] - except: - return QModelIndex() - index = self.createIndex(row, column, pointer) - else: - index = QModelIndex() - return index - - def parent(self, index): - """ - return index of parent element - only valid for links - """ - if index == QModelIndex(): - return QModelIndex() - if index.isValid(): - link = index.internalPointer() - if isinstance(link, Link): - for k, pack in enumerate(self._data): - if pack == link.package: - return self.createIndex(k, 0, link.package) - return QModelIndex() - - def rowCount(self, parent=QModelIndex()): - """ - returns row count for the element - """ - if parent == QModelIndex(): - #return package count - return len(self._data) - else: - if parent.isValid(): - #index is valid - pack = parent.internalPointer() - if isinstance(pack, Package): - #index points to a package - #return len of children - return len(pack.children) - else: - #index is invalid - return False - #files have no children - return 0 - - def columnCount(self, parent=QModelIndex()): - return self.cols - - def hasChildren(self, parent=QModelIndex()): - if not parent.isValid(): - return True - return self.rowCount(parent) > 0 - - def canFetchMore(self, parent): - return False - - def headerData(self, section, orientation, role=Qt.DisplayRole): - """ - returns column heading - """ - if orientation == Qt.Horizontal and role == Qt.DisplayRole: - if section == 0: - return QVariant(_("Name")) - elif section == 1: - return QVariant(_("Plugin")) - elif section == 2: - return QVariant(_("Status")) - elif section == 3: - return QVariant(_("Size")) - return QVariant() - - def flags(self, index): - """ - cell flags - """ - if index.column() == 0 and self.parent(index) == QModelIndex(): - return Qt.ItemIsSelectable | Qt.ItemIsEditable | Qt.ItemIsEnabled - return Qt.ItemIsSelectable | Qt.ItemIsEnabled - - def setData(self, index, value, role=Qt.EditRole): - """ - called if package name editing is finished, sets new name - """ - if index.column() == 0 and self.parent(index) == QModelIndex() and role == Qt.EditRole: - self.connector.setPackageName(index.internalPointer().id, str(value.toString())) - return True - -class Package(object): - """ - package object in the model - """ - - def __init__(self, pack): - self.id = pack.pid - self.children = [] - for f in pack.links: - self.addChild(f) - self.data = {} - self.update(pack) - - def update(self, pack): - """ - update data dict from thift object - """ - data = { - "name": pack.name, - "folder": pack.folder, - "site": pack.site, - "password": pack.password, - "order": pack.order, - } - self.data.update(data) - - def addChild(self, f): - """ - add child (Link) to package - """ - self.children.insert(f.order, Link(f, self)) - self.children = sorted(self.children, key=lambda l: l.data["order"]) - - def getChild(self, fid): - """ - get child from package - """ - for child in self.children: - if child.id == int(fid): - return child - return None - - def getChildKey(self, fid): - """ - get child index - """ - for k, child in enumerate(self.children): - if child.id == int(fid): - return k - return None - - def removeChild(self, fid): - """ - remove child - """ - for k, child in enumerate(self.children): - if child.id == int(fid): - del self.children[k] - -class Link(object): - def __init__(self, f, pack): - self.data = {"downloading": None} - self.update(f) - self.id = f.fid - self.package = pack - - def update(self, f): - """ - update data dict from thift object - """ - data = { - "url": f.url, - "name": f.name, - "plugin": f.plugin, - "size": f.size, - "format_size": f.format_size, - "status": f.status, - "statusmsg": f.statusmsg, - "package": f.packageID, - "error": f.error, - "order": f.order, - } - self.data.update(data) - -class CollectorView(QTreeView): - """ - view component for collector - """ - - def __init__(self, connector): - QTreeView.__init__(self) - self.setModel(CollectorModel(self, connector)) - self.setColumnWidth(0, 500) - self.setColumnWidth(1, 100) - self.setColumnWidth(2, 200) - self.setColumnWidth(3, 100) - - self.setEditTriggers(QAbstractItemView.DoubleClicked | QAbstractItemView.EditKeyPressed) - diff --git a/module/gui/ConnectionManager.py b/module/gui/ConnectionManager.py deleted file mode 100644 index def575abc..000000000 --- a/module/gui/ConnectionManager.py +++ /dev/null @@ -1,302 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from PyQt4.QtCore import * -from PyQt4.QtGui import * - -from os.path import join - -from uuid import uuid4 as uuid - -class ConnectionManager(QWidget): - - - warningShown = False - - def __init__(self): - QWidget.__init__(self) - - if not self.warningShown: - QMessageBox.warning(self, 'Warning', - "We are sorry but the GUI is not stable yet. Please use the webinterface for much better experience. \n", QMessageBox.Ok) - ConnectionManager.warningShown = True - - mainLayout = QHBoxLayout() - buttonLayout = QVBoxLayout() - - self.setWindowTitle(_("pyLoad ConnectionManager")) - self.setWindowIcon(QIcon(join(pypath, "icons","logo.png"))) - - connList = QListWidget() - - new = QPushButton(_("New")) - edit = QPushButton(_("Edit")) - remove = QPushButton(_("Remove")) - connect = QPushButton(_("Connect")) - - #box = QFrame() - boxLayout = QVBoxLayout() - #box.setLayout(boxLayout) - - boxLayout.addWidget(QLabel(_("Connect:"))) - boxLayout.addWidget(connList) - - line = QFrame() - #line.setFixedWidth(100) - line.setFrameShape(line.HLine) - line.setFrameShadow(line.Sunken) - line.setFixedHeight(10) - - boxLayout.addWidget(line) - - form = QFormLayout() - form.setMargin(5) - form.setSpacing(20) - - form.setAlignment(Qt.AlignRight) - checkbox = QCheckBox() - form.addRow(_("Use internal Core:"), checkbox) - - boxLayout.addLayout(form) - - mainLayout.addLayout(boxLayout) - mainLayout.addLayout(buttonLayout) - - buttonLayout.addWidget(new) - buttonLayout.addWidget(edit) - buttonLayout.addWidget(remove) - buttonLayout.addStretch() - buttonLayout.addWidget(connect) - - self.setLayout(mainLayout) - - self.internal = checkbox - self.new = new - self.connectb = connect - self.remove = remove - self.editb = edit - self.connList = connList - self.edit = self.EditWindow() - self.connectSignals() - - self.defaultStates = {} - - def connectSignals(self): - self.connect(self, SIGNAL("setConnections"), self.setConnections) - self.connect(self.new, SIGNAL("clicked()"), self.slotNew) - self.connect(self.editb, SIGNAL("clicked()"), self.slotEdit) - self.connect(self.remove, SIGNAL("clicked()"), self.slotRemove) - self.connect(self.connectb, SIGNAL("clicked()"), self.slotConnect) - self.connect(self.edit, SIGNAL("save"), self.slotSave) - self.connect(self.connList, SIGNAL("itemDoubleClicked(QListWidgetItem *)"), self.slotItemDoubleClicked) - self.connect(self.internal, SIGNAL("clicked()"), self.slotInternal) - - def setConnections(self, connections): - self.connList.clear() - for conn in connections: - item = QListWidgetItem() - item.setData(Qt.DisplayRole, QVariant(conn["name"])) - item.setData(Qt.UserRole, QVariant(conn)) - self.connList.addItem(item) - if conn["default"]: - item.setData(Qt.DisplayRole, QVariant(_("%s (Default)") % conn["name"])) - self.connList.setCurrentItem(item) - - def slotNew(self): - data = {"id":uuid().hex, "type":"remote", "default":False, "name":"", "host":"", "port":"7228", "user":"admin", "password":""} - self.edit.setData(data) - self.edit.show() - - def slotEdit(self): - item = self.connList.currentItem() - data = item.data(Qt.UserRole).toPyObject() - data = self.cleanDict(data) - self.edit.setData(data) - self.edit.show() - - def slotRemove(self): - item = self.connList.currentItem() - data = item.data(Qt.UserRole).toPyObject() - data = self.cleanDict(data) - self.emit(SIGNAL("removeConnection"), data) - - def slotConnect(self): - if self.internal.checkState() == 2: - data = {"type": "internal"} - self.emit(SIGNAL("connect"), data) - else: - item = self.connList.currentItem() - data = item.data(Qt.UserRole).toPyObject() - data = self.cleanDict(data) - self.emit(SIGNAL("connect"), data) - - def cleanDict(self, data): - tmp = {} - for k, d in data.items(): - tmp[str(k)] = d - return tmp - - def slotSave(self, data): - self.emit(SIGNAL("saveConnection"), data) - - def slotItemDoubleClicked(self, defaultItem): - data = defaultItem.data(Qt.UserRole).toPyObject() - self.setDefault(data, True) - did = self.cleanDict(data)["id"] - #allItems = self.connList.findItems("*", Qt.MatchWildcard) - count = self.connList.count() - for i in range(count): - item = self.connList.item(i) - data = item.data(Qt.UserRole).toPyObject() - if self.cleanDict(data)["id"] == did: - continue - self.setDefault(data, False) - - def slotInternal(self): - if self.internal.checkState() == 2: - self.connList.clearSelection() - - def setDefault(self, data, state): - data = self.cleanDict(data) - self.edit.setData(data) - data = self.edit.getData() - data["default"] = state - self.edit.emit(SIGNAL("save"), data) - - class EditWindow(QWidget): - def __init__(self): - QWidget.__init__(self) - - self.setWindowTitle(_("pyLoad ConnectionManager")) - self.setWindowIcon(QIcon(join(pypath, "icons","logo.png"))) - - grid = QGridLayout() - - nameLabel = QLabel(_("Name:")) - hostLabel = QLabel(_("Host:")) - localLabel = QLabel(_("Local:")) - userLabel = QLabel(_("User:")) - pwLabel = QLabel(_("Password:")) - portLabel = QLabel(_("Port:")) - - name = QLineEdit() - host = QLineEdit() - local = QCheckBox() - user = QLineEdit() - password = QLineEdit() - password.setEchoMode(QLineEdit.Password) - port = QSpinBox() - port.setRange(1,10000) - - save = QPushButton(_("Save")) - cancel = QPushButton(_("Cancel")) - - grid.addWidget(nameLabel, 0, 0) - grid.addWidget(name, 0, 1) - grid.addWidget(localLabel, 1, 0) - grid.addWidget(local, 1, 1) - grid.addWidget(hostLabel, 2, 0) - grid.addWidget(host, 2, 1) - grid.addWidget(portLabel, 3, 0) - grid.addWidget(port, 3, 1) - grid.addWidget(userLabel, 4, 0) - grid.addWidget(user, 4, 1) - grid.addWidget(pwLabel, 5, 0) - grid.addWidget(password, 5, 1) - grid.addWidget(cancel, 6, 0) - grid.addWidget(save, 6, 1) - - self.setLayout(grid) - self.controls = {"name": name, - "host": host, - "local": local, - "user": user, - "password": password, - "port": port, - "save": save, - "cancel": cancel} - - self.connect(cancel, SIGNAL("clicked()"), self.hide) - self.connect(save, SIGNAL("clicked()"), self.slotDone) - self.connect(local, SIGNAL("stateChanged(int)"), self.slotLocalChanged) - - self.id = None - self.default = None - - def setData(self, data): - if not data: return - - self.id = data["id"] - self.default = data["default"] - self.controls["name"].setText(data["name"]) - if data["type"] == "local": - data["local"] = True - else: - data["local"] = False - self.controls["local"].setChecked(data["local"]) - if not data["local"]: - self.controls["user"].setText(data["user"]) - self.controls["password"].setText(data["password"]) - self.controls["port"].setValue(int(data["port"])) - self.controls["host"].setText(data["host"]) - self.controls["user"].setDisabled(False) - self.controls["password"].setDisabled(False) - self.controls["port"].setDisabled(False) - self.controls["host"].setDisabled(False) - else: - self.controls["user"].setText("") - self.controls["port"].setValue(1) - self.controls["host"].setText("") - self.controls["user"].setDisabled(True) - self.controls["password"].setDisabled(True) - self.controls["port"].setDisabled(True) - self.controls["host"].setDisabled(True) - - def slotLocalChanged(self, val): - if val == 2: - self.controls["user"].setDisabled(True) - self.controls["password"].setDisabled(True) - self.controls["port"].setDisabled(True) - self.controls["host"].setDisabled(True) - elif val == 0: - self.controls["user"].setDisabled(False) - self.controls["password"].setDisabled(False) - self.controls["port"].setDisabled(False) - self.controls["host"].setDisabled(False) - - def getData(self): - d = {} - d["id"] = self.id - d["default"] = self.default - d["name"] = self.controls["name"].text() - d["local"] = self.controls["local"].isChecked() - d["user"] = self.controls["user"].text() - d["password"] = self.controls["password"].text() - d["host"] = self.controls["host"].text() - d["port"] = self.controls["port"].value() - if d["local"]: - d["type"] = "local" - else: - d["type"] = "remote" - return d - - def slotDone(self): - data = self.getData() - self.hide() - self.emit(SIGNAL("save"), data) - diff --git a/module/gui/CoreConfigParser.py b/module/gui/CoreConfigParser.py deleted file mode 100644 index 0d1d298c6..000000000 --- a/module/gui/CoreConfigParser.py +++ /dev/null @@ -1,165 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import with_statement -from os.path import exists -from os.path import join - - -CONF_VERSION = 1 - -######################################################################## -class ConfigParser: - - #---------------------------------------------------------------------- - def __init__(self, configdir): - """Constructor""" - self.configdir = configdir - self.config = {} - - if self.checkVersion(): - self.readConfig() - - #---------------------------------------------------------------------- - def checkVersion(self): - - if not exists(join(self.configdir, "pyload.conf")): - return False - f = open(join(self.configdir, "pyload.conf"), "rb") - v = f.readline() - f.close() - v = v[v.find(":")+1:].strip() - - if int(v) < CONF_VERSION: - return False - - return True - - #---------------------------------------------------------------------- - def readConfig(self): - """reads the config file""" - - self.config = self.parseConfig(join(self.configdir, "pyload.conf")) - - - #---------------------------------------------------------------------- - def parseConfig(self, config): - """parses a given configfile""" - - f = open(config) - - config = f.read() - - config = config.split("\n")[1:] - - conf = {} - - section, option, value, typ, desc = "","","","","" - - listmode = False - - for line in config: - - line = line.rpartition("#") # removes comments - - if line[1]: - line = line[0] - else: - line = line[2] - - line = line.strip() - - try: - - if line == "": - continue - elif line.endswith(":"): - section, none, desc = line[:-1].partition('-') - section = section.strip() - desc = desc.replace('"', "").strip() - conf[section] = { "desc" : desc } - else: - if listmode: - - if line.endswith("]"): - listmode = False - line = line.replace("]","") - - value += [self.cast(typ, x.strip()) for x in line.split(",") if x] - - if not listmode: - conf[section][option] = { "desc" : desc, - "type" : typ, - "value" : value} - - - else: - content, none, value = line.partition("=") - - content, none, desc = content.partition(":") - - desc = desc.replace('"', "").strip() - - typ, option = content.split() - - value = value.strip() - - if value.startswith("["): - if value.endswith("]"): - listmode = False - value = value[:-1] - else: - listmode = True - - value = [self.cast(typ, x.strip()) for x in value[1:].split(",") if x] - else: - value = self.cast(typ, value) - - if not listmode: - conf[section][option] = { "desc" : desc, - "type" : typ, - "value" : value} - - except: - pass - - - f.close() - return conf - - #---------------------------------------------------------------------- - def cast(self, typ, value): - """cast value to given format""" - if type(value) not in (str, unicode): - return value - - if typ == "int": - return int(value) - elif typ == "bool": - return True if value.lower() in ("1","true", "on", "an","yes") else False - else: - return value - - #---------------------------------------------------------------------- - def get(self, section, option): - """get value""" - return self.config[section][option]["value"] - - #---------------------------------------------------------------------- - def __getitem__(self, section): - """provides dictonary like access: c['section']['option']""" - return Section(self, section) - -######################################################################## -class Section: - """provides dictionary like access for configparser""" - - #---------------------------------------------------------------------- - def __init__(self, parser, section): - """Constructor""" - self.parser = parser - self.section = section - - #---------------------------------------------------------------------- - def __getitem__(self, item): - """getitem""" - return self.parser.get(self.section, item) diff --git a/module/gui/LinkDock.py b/module/gui/LinkDock.py deleted file mode 100644 index ac2d4aae5..000000000 --- a/module/gui/LinkDock.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from PyQt4.QtCore import * -from PyQt4.QtGui import * - -class NewLinkDock(QDockWidget): - def __init__(self): - QDockWidget.__init__(self, "New Links") - self.setObjectName("New Links Dock") - self.widget = NewLinkWindow(self) - self.setWidget(self.widget) - self.setAllowedAreas(Qt.RightDockWidgetArea|Qt.LeftDockWidgetArea) - self.hide() - - def slotDone(self): - text = str(self.widget.box.toPlainText()) - lines = text.splitlines() - self.emit(SIGNAL("done"), lines) - self.widget.box.clear() - self.hide() - -class NewLinkWindow(QWidget): - def __init__(self, dock): - QWidget.__init__(self) - self.dock = dock - self.setLayout(QVBoxLayout()) - layout = self.layout() - - explanationLabel = QLabel("Select a package and then click Add button.") - boxLabel = QLabel("Paste URLs here:") - self.box = QTextEdit() - - save = QPushButton("Add") - - layout.addWidget(explanationLabel) - layout.addWidget(boxLabel) - layout.addWidget(self.box) - layout.addWidget(save) - - self.connect(save, SIGNAL("clicked()"), self.dock.slotDone) diff --git a/module/gui/MainWindow.py b/module/gui/MainWindow.py deleted file mode 100644 index c71112e9b..000000000 --- a/module/gui/MainWindow.py +++ /dev/null @@ -1,697 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from PyQt4.QtCore import * -from PyQt4.QtGui import * - -from os.path import join - -from module.gui.PackageDock import * -from module.gui.LinkDock import * -from module.gui.CaptchaDock import CaptchaDock -from module.gui.SettingsWidget import SettingsWidget - -from module.gui.Collector import CollectorView, Package, Link -from module.gui.Queue import QueueView -from module.gui.Overview import OverviewView -from module.gui.Accounts import AccountView -from module.gui.AccountEdit import AccountEdit - -from module.remote.thriftbackend.ThriftClient import AccountInfo - -class MainWindow(QMainWindow): - def __init__(self, connector): - """ - set up main window - """ - QMainWindow.__init__(self) - #window stuff - self.setWindowTitle(_("pyLoad Client")) - self.setWindowIcon(QIcon(join(pypath, "icons","logo.png"))) - self.resize(1000,600) - - #layout version - self.version = 3 - - #init docks - self.newPackDock = NewPackageDock() - self.addDockWidget(Qt.RightDockWidgetArea, self.newPackDock) - self.connect(self.newPackDock, SIGNAL("done"), self.slotAddPackage) - self.captchaDock = CaptchaDock() - self.addDockWidget(Qt.BottomDockWidgetArea, self.captchaDock) - self.newLinkDock = NewLinkDock() - self.addDockWidget(Qt.RightDockWidgetArea, self.newLinkDock) - self.connect(self.newLinkDock, SIGNAL("done"), self.slotAddLinksToPackage) - - #central widget, layout - self.masterlayout = QVBoxLayout() - lw = QWidget() - lw.setLayout(self.masterlayout) - self.setCentralWidget(lw) - - #status - self.statusw = QFrame() - self.statusw.setFrameStyle(QFrame.StyledPanel | QFrame.Raised) - self.statusw.setLineWidth(2) - self.statusw.setLayout(QGridLayout()) - #palette = self.statusw.palette() - #palette.setColor(QPalette.Window, QColor(255, 255, 255)) - #self.statusw.setPalette(palette) - #self.statusw.setAutoFillBackground(True) - l = self.statusw.layout() - - class BoldLabel(QLabel): - def __init__(self, text): - QLabel.__init__(self, text) - f = self.font() - f.setBold(True) - self.setFont(f) - self.setAlignment(Qt.AlignRight) - - class Seperator(QFrame): - def __init__(self): - QFrame.__init__(self) - self.setFrameShape(QFrame.VLine) - self.setFrameShadow(QFrame.Sunken) - - l.addWidget(BoldLabel(_("Packages:")), 0, 0) - self.packageCount = QLabel("0") - l.addWidget(self.packageCount, 0, 1) - - l.addWidget(BoldLabel(_("Files:")), 0, 2) - self.fileCount = QLabel("0") - l.addWidget(self.fileCount, 0, 3) - - l.addWidget(BoldLabel(_("Status:")), 0, 4) - self.status = QLabel("running") - l.addWidget(self.status, 0, 5) - - l.addWidget(BoldLabel(_("Space:")), 0, 6) - self.space = QLabel("") - l.addWidget(self.space, 0, 7) - - l.addWidget(BoldLabel(_("Speed:")), 0, 8) - self.speed = QLabel("") - l.addWidget(self.speed, 0, 9) - - #l.addWidget(BoldLabel(_("Max. downloads:")), 0, 9) - #l.addWidget(BoldLabel(_("Max. chunks:")), 1, 9) - #self.maxDownloads = QSpinBox() - #self.maxDownloads.setEnabled(False) - #self.maxChunks = QSpinBox() - #self.maxChunks.setEnabled(False) - #l.addWidget(self.maxDownloads, 0, 10) - #l.addWidget(self.maxChunks, 1, 10) - - #set menubar and statusbar - self.menubar = self.menuBar() - #self.statusbar = self.statusBar() - #self.connect(self.statusbar, SIGNAL("showMsg"), self.statusbar.showMessage) - #self.serverStatus = QLabel(_("Status: Not Connected")) - #self.statusbar.addPermanentWidget(self.serverStatus) - - #menu - self.menus = {"file": self.menubar.addMenu(_("File")), - "connections": self.menubar.addMenu(_("Connections"))} - - #menu actions - self.mactions = {"exit": QAction(_("Exit"), self.menus["file"]), - "manager": QAction(_("Connection manager"), self.menus["connections"])} - - #add menu actions - self.menus["file"].addAction(self.mactions["exit"]) - self.menus["connections"].addAction(self.mactions["manager"]) - - #toolbar - self.actions = {} - self.init_toolbar() - - #tabs - self.tabw = QTabWidget() - self.tabs = {"overview": {"w": QWidget()}, - "queue": {"w": QWidget()}, - "collector": {"w": QWidget()}, - "accounts": {"w": QWidget()}, - "settings": {}} - #self.tabs["settings"]["s"] = QScrollArea() - self.tabs["settings"]["w"] = SettingsWidget() - #self.tabs["settings"]["s"].setWidgetResizable(True) - #self.tabs["settings"]["s"].setWidget(self.tabs["settings"]["w"]) - self.tabs["log"] = {"w":QWidget()} - self.tabw.addTab(self.tabs["overview"]["w"], _("Overview")) - self.tabw.addTab(self.tabs["queue"]["w"], _("Queue")) - self.tabw.addTab(self.tabs["collector"]["w"], _("Collector")) - self.tabw.addTab(self.tabs["accounts"]["w"], _("Accounts")) - self.tabw.addTab(self.tabs["settings"]["w"], _("Settings")) - self.tabw.addTab(self.tabs["log"]["w"], _("Log")) - - #init tabs - self.init_tabs(connector) - - #context menus - self.init_context() - - #layout - self.masterlayout.addWidget(self.tabw) - self.masterlayout.addWidget(self.statusw) - - #signals.. - self.connect(self.mactions["manager"], SIGNAL("triggered()"), self.slotShowConnector) - - self.connect(self.tabs["queue"]["view"], SIGNAL('customContextMenuRequested(const QPoint &)'), self.slotQueueContextMenu) - self.connect(self.tabs["collector"]["package_view"], SIGNAL('customContextMenuRequested(const QPoint &)'), self.slotCollectorContextMenu) - self.connect(self.tabs["accounts"]["view"], SIGNAL('customContextMenuRequested(const QPoint &)'), self.slotAccountContextMenu) - - self.connect(self.tabw, SIGNAL("currentChanged(int)"), self.slotTabChanged) - - self.lastAddedID = None - - self.connector = connector - - def init_toolbar(self): - """ - create toolbar - """ - self.toolbar = self.addToolBar(_("Hide Toolbar")) - self.toolbar.setObjectName("Main Toolbar") - self.toolbar.setIconSize(QSize(30,30)) - self.toolbar.setMovable(False) - self.actions["toggle_status"] = self.toolbar.addAction(_("Toggle Pause/Resume")) - pricon = QIcon() - pricon.addFile(join(pypath, "icons","toolbar_start.png"), QSize(), QIcon.Normal, QIcon.Off) - pricon.addFile(join(pypath, "icons","toolbar_pause.png"), QSize(), QIcon.Normal, QIcon.On) - self.actions["toggle_status"].setIcon(pricon) - self.actions["toggle_status"].setCheckable(True) - self.actions["status_stop"] = self.toolbar.addAction(QIcon(join(pypath, "icons","toolbar_stop.png")), _("Stop")) - self.toolbar.addSeparator() - self.actions["add"] = self.toolbar.addAction(QIcon(join(pypath, "icons","toolbar_add.png")), _("Add")) - self.toolbar.addSeparator() - self.actions["clipboard"] = self.toolbar.addAction(QIcon(join(pypath, "icons","clipboard.png")), _("Check Clipboard")) - self.actions["clipboard"].setCheckable(True) - - self.connect(self.actions["toggle_status"], SIGNAL("toggled(bool)"), self.slotToggleStatus) - self.connect(self.actions["clipboard"], SIGNAL("toggled(bool)"), self.slotToggleClipboard) - self.connect(self.actions["status_stop"], SIGNAL("triggered()"), self.slotStatusStop) - self.addMenu = QMenu() - packageAction = self.addMenu.addAction(_("Package")) - containerAction = self.addMenu.addAction(_("Container")) - accountAction = self.addMenu.addAction(_("Account")) - linksAction = self.addMenu.addAction(_("Links")) - self.connect(self.actions["add"], SIGNAL("triggered()"), self.slotAdd) - self.connect(packageAction, SIGNAL("triggered()"), self.slotShowAddPackage) - self.connect(containerAction, SIGNAL("triggered()"), self.slotShowAddContainer) - self.connect(accountAction, SIGNAL("triggered()"), self.slotNewAccount) - self.connect(linksAction, SIGNAL("triggered()"), self.slotShowAddLinks) - - def init_tabs(self, connector): - """ - create tabs - """ - #overview - self.tabs["overview"]["l"] = QGridLayout() - self.tabs["overview"]["w"].setLayout(self.tabs["overview"]["l"]) - self.tabs["overview"]["view"] = OverviewView(connector) - self.tabs["overview"]["l"].addWidget(self.tabs["overview"]["view"]) - - #queue - self.tabs["queue"]["l"] = QGridLayout() - self.tabs["queue"]["w"].setLayout(self.tabs["queue"]["l"]) - self.tabs["queue"]["view"] = QueueView(connector) - self.tabs["queue"]["l"].addWidget(self.tabs["queue"]["view"]) - - #collector - toQueue = QPushButton(_("Push selected packages to queue")) - self.tabs["collector"]["l"] = QGridLayout() - self.tabs["collector"]["w"].setLayout(self.tabs["collector"]["l"]) - self.tabs["collector"]["package_view"] = CollectorView(connector) - self.tabs["collector"]["l"].addWidget(self.tabs["collector"]["package_view"], 0, 0) - self.tabs["collector"]["l"].addWidget(toQueue, 1, 0) - self.connect(toQueue, SIGNAL("clicked()"), self.slotPushPackageToQueue) - self.tabs["collector"]["package_view"].setContextMenuPolicy(Qt.CustomContextMenu) - self.tabs["queue"]["view"].setContextMenuPolicy(Qt.CustomContextMenu) - - #log - self.tabs["log"]["l"] = QGridLayout() - self.tabs["log"]["w"].setLayout(self.tabs["log"]["l"]) - self.tabs["log"]["text"] = QTextEdit() - self.tabs["log"]["text"].logOffset = 0 - self.tabs["log"]["text"].setReadOnly(True) - self.connect(self.tabs["log"]["text"], SIGNAL("append(QString)"), self.tabs["log"]["text"].append) - self.tabs["log"]["l"].addWidget(self.tabs["log"]["text"]) - - #accounts - self.tabs["accounts"]["view"] = AccountView(connector) - self.tabs["accounts"]["w"].setLayout(QVBoxLayout()) - self.tabs["accounts"]["w"].layout().addWidget(self.tabs["accounts"]["view"]) - newbutton = QPushButton(_("New Account")) - self.tabs["accounts"]["w"].layout().addWidget(newbutton) - self.connect(newbutton, SIGNAL("clicked()"), self.slotNewAccount) - self.tabs["accounts"]["view"].setContextMenuPolicy(Qt.CustomContextMenu) - - def init_context(self): - """ - create context menus - """ - self.activeMenu = None - #queue - self.queueContext = QMenu() - self.queueContext.buttons = {} - self.queueContext.item = (None, None) - self.queueContext.buttons["remove"] = QAction(QIcon(join(pypath, "icons","remove_small.png")), _("Remove"), self.queueContext) - self.queueContext.buttons["restart"] = QAction(QIcon(join(pypath, "icons","refresh_small.png")), _("Restart"), self.queueContext) - self.queueContext.buttons["pull"] = QAction(QIcon(join(pypath, "icons","pull_small.png")), _("Pull out"), self.queueContext) - self.queueContext.buttons["abort"] = QAction(QIcon(join(pypath, "icons","abort.png")), _("Abort"), self.queueContext) - self.queueContext.buttons["edit"] = QAction(QIcon(join(pypath, "icons","edit_small.png")), _("Edit Name"), self.queueContext) - self.queueContext.addAction(self.queueContext.buttons["pull"]) - self.queueContext.addAction(self.queueContext.buttons["edit"]) - self.queueContext.addAction(self.queueContext.buttons["remove"]) - self.queueContext.addAction(self.queueContext.buttons["restart"]) - self.queueContext.addAction(self.queueContext.buttons["abort"]) - self.connect(self.queueContext.buttons["remove"], SIGNAL("triggered()"), self.slotRemoveDownload) - self.connect(self.queueContext.buttons["restart"], SIGNAL("triggered()"), self.slotRestartDownload) - self.connect(self.queueContext.buttons["pull"], SIGNAL("triggered()"), self.slotPullOutPackage) - self.connect(self.queueContext.buttons["abort"], SIGNAL("triggered()"), self.slotAbortDownload) - self.connect(self.queueContext.buttons["edit"], SIGNAL("triggered()"), self.slotEditPackage) - - #collector - self.collectorContext = QMenu() - self.collectorContext.buttons = {} - self.collectorContext.item = (None, None) - self.collectorContext.buttons["remove"] = QAction(QIcon(join(pypath, "icons","remove_small.png")), _("Remove"), self.collectorContext) - self.collectorContext.buttons["push"] = QAction(QIcon(join(pypath, "icons","push_small.png")), _("Push to queue"), self.collectorContext) - self.collectorContext.buttons["edit"] = QAction(QIcon(join(pypath, "icons","edit_small.png")), _("Edit Name"), self.collectorContext) - self.collectorContext.buttons["restart"] = QAction(QIcon(join(pypath, "icons","refresh_small.png")), _("Restart"), self.collectorContext) - self.collectorContext.buttons["refresh"] = QAction(QIcon(join(pypath, "icons","refresh1_small.png")),_("Refresh Status"), self.collectorContext) - self.collectorContext.addAction(self.collectorContext.buttons["push"]) - self.collectorContext.addSeparator() - self.collectorContext.buttons["add"] = self.collectorContext.addMenu(QIcon(join(pypath, "icons","add_small.png")), _("Add")) - self.collectorContext.addAction(self.collectorContext.buttons["edit"]) - self.collectorContext.addAction(self.collectorContext.buttons["remove"]) - self.collectorContext.addAction(self.collectorContext.buttons["restart"]) - self.collectorContext.addSeparator() - self.collectorContext.addAction(self.collectorContext.buttons["refresh"]) - packageAction = self.collectorContext.buttons["add"].addAction(_("Package")) - containerAction = self.collectorContext.buttons["add"].addAction(_("Container")) - linkAction = self.collectorContext.buttons["add"].addAction(_("Links")) - self.connect(self.collectorContext.buttons["remove"], SIGNAL("triggered()"), self.slotRemoveDownload) - self.connect(self.collectorContext.buttons["push"], SIGNAL("triggered()"), self.slotPushPackageToQueue) - self.connect(self.collectorContext.buttons["edit"], SIGNAL("triggered()"), self.slotEditPackage) - self.connect(self.collectorContext.buttons["restart"], SIGNAL("triggered()"), self.slotRestartDownload) - self.connect(self.collectorContext.buttons["refresh"], SIGNAL("triggered()"), self.slotRefreshPackage) - self.connect(packageAction, SIGNAL("triggered()"), self.slotShowAddPackage) - self.connect(containerAction, SIGNAL("triggered()"), self.slotShowAddContainer) - self.connect(linkAction, SIGNAL("triggered()"), self.slotShowAddLinks) - - self.accountContext = QMenu() - self.accountContext.buttons = {} - self.accountContext.buttons["add"] = QAction(QIcon(join(pypath, "icons","add_small.png")), _("Add"), self.accountContext) - self.accountContext.buttons["remove"] = QAction(QIcon(join(pypath, "icons","remove_small.png")), _("Remove"), self.accountContext) - self.accountContext.buttons["edit"] = QAction(QIcon(join(pypath, "icons","edit_small.png")), _("Edit"), self.accountContext) - self.accountContext.addAction(self.accountContext.buttons["add"]) - self.accountContext.addAction(self.accountContext.buttons["edit"]) - self.accountContext.addAction(self.accountContext.buttons["remove"]) - self.connect(self.accountContext.buttons["add"], SIGNAL("triggered()"), self.slotNewAccount) - self.connect(self.accountContext.buttons["edit"], SIGNAL("triggered()"), self.slotEditAccount) - self.connect(self.accountContext.buttons["remove"], SIGNAL("triggered()"), self.slotRemoveAccount) - - def slotToggleStatus(self, status): - """ - pause/start toggle (toolbar) - """ - self.emit(SIGNAL("setDownloadStatus"), status) - - def slotStatusStop(self): - """ - stop button (toolbar) - """ - self.emit(SIGNAL("stopAllDownloads")) - - def slotAdd(self): - """ - add button (toolbar) - show context menu (choice: links/package) - """ - self.addMenu.exec_(QCursor.pos()) - - def slotShowAddPackage(self): - """ - action from add-menu - show new-package dock - """ - self.tabw.setCurrentIndex(1) - self.newPackDock.show() - - def slotShowAddLinks(self): - """ - action from add-menu - show new-links dock - """ - self.tabw.setCurrentIndex(1) - self.newLinkDock.show() - - def slotShowConnector(self): - """ - connectionmanager action triggered - let main to the stuff - """ - self.emit(SIGNAL("connector")) - - def slotAddPackage(self, name, links, password=None): - """ - new package - let main to the stuff - """ - self.emit(SIGNAL("addPackage"), name, links, password) - - def slotAddLinksToPackage(self, links): - """ - adds links to currently selected package - only in collector - """ - if self.tabw.currentIndex() != 1: - return - - smodel = self.tabs["collector"]["package_view"].selectionModel() - for index in smodel.selectedRows(0): - item = index.internalPointer() - if isinstance(item, Package): - self.connector.proxy.addFiles(item.id, links) - break - - def slotShowAddContainer(self): - """ - action from add-menu - show file selector, emit upload - """ - typeStr = ";;".join([ - _("All Container Types (%s)") % "*.dlc *.ccf *.rsdf *.txt", - _("DLC (%s)") % "*.dlc", - _("CCF (%s)") % "*.ccf", - _("RSDF (%s)") % "*.rsdf", - _("Text Files (%s)") % "*.txt" - ]) - fileNames = QFileDialog.getOpenFileNames(self, _("Open container"), "", typeStr) - for name in fileNames: - self.emit(SIGNAL("addContainer"), str(name)) - - def slotPushPackageToQueue(self): - """ - push collector pack to queue - get child ids - let main to the rest - """ - smodel = self.tabs["collector"]["package_view"].selectionModel() - for index in smodel.selectedRows(0): - item = index.internalPointer() - if isinstance(item, Package): - self.emit(SIGNAL("pushPackageToQueue"), item.id) - else: - self.emit(SIGNAL("pushPackageToQueue"), item.package.id) - - def saveWindow(self): - """ - get window state/geometry - pass data to main - """ - state_raw = self.saveState(self.version) - geo_raw = self.saveGeometry() - - state = str(state_raw.toBase64()) - geo = str(geo_raw.toBase64()) - - self.emit(SIGNAL("saveMainWindow"), state, geo) - - def closeEvent(self, event): - """ - somebody wants to close me! - let me first save my state - """ - self.saveWindow() - event.ignore() - self.hide() - self.emit(SIGNAL("hidden")) - - # quit when no tray is available - if not QSystemTrayIcon.isSystemTrayAvailable(): - self.emit(SIGNAL("Quit")) - - def restoreWindow(self, state, geo): - """ - restore window state/geometry - """ - state = QByteArray(state) - geo = QByteArray(geo) - - state_raw = QByteArray.fromBase64(state) - geo_raw = QByteArray.fromBase64(geo) - - self.restoreState(state_raw, self.version) - self.restoreGeometry(geo_raw) - - def slotQueueContextMenu(self, pos): - """ - custom context menu in queue view requested - """ - globalPos = self.tabs["queue"]["view"].mapToGlobal(pos) - i = self.tabs["queue"]["view"].indexAt(pos) - if not i: - return - item = i.internalPointer() - menuPos = QCursor.pos() - menuPos.setX(menuPos.x()+2) - self.activeMenu = self.queueContext - showAbort = False - if isinstance(item, Link) and item.data["downloading"]: - showAbort = True - elif isinstance(item, Package): - for child in item.children: - if child.data["downloading"]: - showAbort = True - break - if showAbort: - self.queueContext.buttons["abort"].setEnabled(True) - else: - self.queueContext.buttons["abort"].setEnabled(False) - if isinstance(item, Package): - self.queueContext.index = i - #self.queueContext.buttons["remove"].setEnabled(True) - #self.queueContext.buttons["restart"].setEnabled(True) - self.queueContext.buttons["pull"].setEnabled(True) - self.queueContext.buttons["edit"].setEnabled(True) - elif isinstance(item, Link): - self.collectorContext.index = i - self.collectorContext.buttons["edit"].setEnabled(False) - self.collectorContext.buttons["remove"].setEnabled(True) - self.collectorContext.buttons["push"].setEnabled(False) - self.collectorContext.buttons["restart"].setEnabled(True) - else: - self.queueContext.index = None - #self.queueContext.buttons["remove"].setEnabled(False) - #self.queueContext.buttons["restart"].setEnabled(False) - self.queueContext.buttons["pull"].setEnabled(False) - self.queueContext.buttons["edit"].setEnabled(False) - self.queueContext.exec_(menuPos) - - def slotCollectorContextMenu(self, pos): - """ - custom context menu in package collector view requested - """ - globalPos = self.tabs["collector"]["package_view"].mapToGlobal(pos) - i = self.tabs["collector"]["package_view"].indexAt(pos) - if not i: - return - item = i.internalPointer() - menuPos = QCursor.pos() - menuPos.setX(menuPos.x()+2) - self.activeMenu = self.collectorContext - if isinstance(item, Package): - self.collectorContext.index = i - self.collectorContext.buttons["edit"].setEnabled(True) - self.collectorContext.buttons["remove"].setEnabled(True) - self.collectorContext.buttons["push"].setEnabled(True) - self.collectorContext.buttons["restart"].setEnabled(True) - elif isinstance(item, Link): - self.collectorContext.index = i - self.collectorContext.buttons["edit"].setEnabled(False) - self.collectorContext.buttons["remove"].setEnabled(True) - self.collectorContext.buttons["push"].setEnabled(False) - self.collectorContext.buttons["restart"].setEnabled(True) - else: - self.collectorContext.index = None - self.collectorContext.buttons["edit"].setEnabled(False) - self.collectorContext.buttons["remove"].setEnabled(False) - self.collectorContext.buttons["push"].setEnabled(False) - self.collectorContext.buttons["restart"].setEnabled(False) - self.collectorContext.exec_(menuPos) - - def slotLinkCollectorContextMenu(self, pos): - """ - custom context menu in link collector view requested - """ - pass - - def slotRestartDownload(self): - """ - restart download action is triggered - """ - smodel = self.tabs["queue"]["view"].selectionModel() - for index in smodel.selectedRows(0): - item = index.internalPointer() - self.emit(SIGNAL("restartDownload"), item.id, isinstance(item, Package)) - - def slotRemoveDownload(self): - """ - remove download action is triggered - """ - if self.activeMenu == self.queueContext: - view = self.tabs["queue"]["view"] - else: - view = self.tabs["collector"]["package_view"] - smodel = view.selectionModel() - for index in smodel.selectedRows(0): - item = index.internalPointer() - self.emit(SIGNAL("removeDownload"), item.id, isinstance(item, Package)) - - def slotToggleClipboard(self, status): - """ - check clipboard (toolbar) - """ - self.emit(SIGNAL("setClipboardStatus"), status) - - def slotEditPackage(self): - # in Queue, only edit name - if self.activeMenu == self.queueContext: - view = self.tabs["queue"]["view"] - else: - view = self.tabs["collector"]["package_view"] - view.edit(self.activeMenu.index) - - def slotEditCommit(self, editor): - self.emit(SIGNAL("changePackageName"), self.activeMenu.index.internalPointer().id, editor.text()) - - def slotPullOutPackage(self): - """ - pull package out of the queue - """ - smodel = self.tabs["queue"]["view"].selectionModel() - for index in smodel.selectedRows(0): - item = index.internalPointer() - if isinstance(item, Package): - self.emit(SIGNAL("pullOutPackage"), item.id) - else: - self.emit(SIGNAL("pullOutPackage"), item.package.id) - - def slotAbortDownload(self): - view = self.tabs["queue"]["view"] - smodel = view.selectionModel() - for index in smodel.selectedRows(0): - item = index.internalPointer() - self.emit(SIGNAL("abortDownload"), item.id, isinstance(item, Package)) - - # TODO disabled because changing desktop on linux, main window disappears - #def changeEvent(self, e): - # if e.type() == QEvent.WindowStateChange and self.isMinimized(): - # e.ignore() - # self.hide() - # self.emit(SIGNAL("hidden")) - # else: - # super(MainWindow, self).changeEvent(e) - - def slotTabChanged(self, index): - if index == 2: - self.emit(SIGNAL("reloadAccounts")) - elif index == 3: - self.tabs["settings"]["w"].loadConfig() - - def slotRefreshPackage(self): - smodel = self.tabs["collector"]["package_view"].selectionModel() - for index in smodel.selectedRows(0): - item = index.internalPointer() - pid = item.id - if isinstance(item, Link): - pid = item.package.id - self.emit(SIGNAL("refreshStatus"), pid) - - def slotNewAccount(self): - types = self.connector.proxy.getAccountTypes() - self.accountEdit = AccountEdit.newAccount(types) - - #TODO make more easy n1, n2, n3 - def save(data): - if data["password"]: - self.accountEdit.close() - n1 = data["acctype"] - n2 = data["login"] - n3 = data["password"] - self.connector.updateAccount(n1, n2, n3, None) - - self.accountEdit.connect(self.accountEdit, SIGNAL("done"), save) - self.accountEdit.show() - - def slotEditAccount(self): - types = self.connector.getAccountTypes() - - data = self.tabs["accounts"]["view"].selectedIndexes() - if len(data) < 1: - return - - data = data[0].internalPointer() - - self.accountEdit = AccountEdit.editAccount(types, data) - - #TODO make more easy n1, n2, n3 - #TODO reload accounts tab after insert of edit account - #TODO if account does not exist give error - def save(data): - self.accountEdit.close() - n1 = data["acctype"] - n2 = data["login"] - if data["password"]: - n3 = data["password"] - self.connector.updateAccount(n1, n2, n3, None) - - self.accountEdit.connect(self.accountEdit, SIGNAL("done"), save) - self.accountEdit.show() - - def slotRemoveAccount(self): - data = self.tabs["accounts"]["view"].selectedIndexes() - if len(data) < 1: - return - - data = data[0].internalPointer() - - self.connector.removeAccount(data.type, data.login) - - def slotAccountContextMenu(self, pos): - globalPos = self.tabs["accounts"]["view"].mapToGlobal(pos) - i = self.tabs["accounts"]["view"].indexAt(pos) - if not i: - return - - data = i.internalPointer() - - if data is None: - self.accountContext.buttons["edit"].setEnabled(False) - self.accountContext.buttons["remove"].setEnabled(False) - else: - self.accountContext.buttons["edit"].setEnabled(True) - self.accountContext.buttons["remove"].setEnabled(True) - - menuPos = QCursor.pos() - menuPos.setX(menuPos.x()+2) - self.accountContext.exec_(menuPos) diff --git a/module/gui/Overview.py b/module/gui/Overview.py deleted file mode 100644 index 183383b5e..000000000 --- a/module/gui/Overview.py +++ /dev/null @@ -1,197 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from PyQt4.QtCore import * -from PyQt4.QtGui import * - -from module.utils import formatSpeed, formatSize - -class OverviewModel(QAbstractListModel): - PackageName = 10 - Progress = 11 - PartsFinished = 12 - Parts = 13 - ETA = 14 - Speed = 15 - CurrentSize = 16 - MaxSize = 17 - Status = 18 - - def __init__(self, view, connector): - QAbstractListModel.__init__(self) - - self.packages = [] - - def queueChanged(self): #dirty.. - self.beginResetModel() - - self.packages = [] - - def partsFinished(p): - f = 0 - for c in p.children: - if c.data["status"] == 0: - f += 1 - return f - - def maxSize(p): - ms = 0 - cs = 0 - for c in p.children: - try: - s = c.data["downloading"]["size"] - except: - s = c.data["size"] - if c.data["downloading"]: - cs += s - c.data["downloading"]["bleft"] - elif self.queue.getProgress(c, False) == 100: - cs += s - ms += s - return ms, cs - - def getProgress(p): - for c in p.children: - if c.data["status"] == 13: - pass # TODO return _("Unpacking"), int(c.data["progress"]) - return _("Downloading"), self.queue.getProgress(p) - - d = self.queue._data - for p in d: - status, progress = getProgress(p) - maxsize, currentsize = maxSize(p) - speed = self.queue.getSpeed(p) - if speed: - eta = (maxsize - (maxsize * (progress/100.0)))/speed - else: - eta = 0 - if not speed and not progress: - status = _("Queued") - info = { - OverviewModel.PackageName: p.data["name"], - OverviewModel.Progress: progress, - OverviewModel.PartsFinished: partsFinished(p), - OverviewModel.Parts: len(p.children), - OverviewModel.ETA: int(eta), - OverviewModel.Speed: speed, - OverviewModel.CurrentSize: currentsize, - OverviewModel.MaxSize: maxsize, - OverviewModel.Status: status, - } - - self.packages.append(info) - - self.endResetModel() - - def headerData(self, section, orientation, role=Qt.DisplayRole): - return QVariant(_("Package")) - - def rowCount(self, parent=QModelIndex()): - return len(self.packages) - - def data(self, index, role=Qt.DisplayRole): - if role in [OverviewModel.PackageName, OverviewModel.Progress, OverviewModel.PartsFinished, OverviewModel.Parts, OverviewModel.ETA, OverviewModel.Speed, OverviewModel.CurrentSize, OverviewModel.MaxSize, OverviewModel.Status]: - return QVariant(self.packages[index.row()][role]) - return QVariant() - -class OverviewView(QListView): - def __init__(self, connector): - QListView.__init__(self) - self.setModel(OverviewModel(self, connector)) - - self.setAlternatingRowColors(True) - self.delegate = OverviewDelegate(self) - self.setItemDelegate(self.delegate) - -class OverviewDelegate(QItemDelegate): - def __init__(self, parent): - QItemDelegate.__init__(self, parent) - self.parent = parent - self.model = parent.model() - - def paint(self, painter, option, index): - option.rect.setHeight(59+16) - option.rect.setWidth(self.parent.width()-20) - - #if option.state & QStyle.State_Selected: - # painter.fillRect(option.rect, option.palette.color(QPalette.Highlight)) - - packagename = index.data(OverviewModel.PackageName).toString() - partsf = index.data(OverviewModel.PartsFinished).toString() - parts = index.data(OverviewModel.Parts).toString() - eta = int(index.data(OverviewModel.ETA).toString()) - speed = index.data(OverviewModel.Speed).toString() or 0 - progress = int(index.data(OverviewModel.Progress).toString()) - currentSize = int(index.data(OverviewModel.CurrentSize).toString()) - maxSize = int(index.data(OverviewModel.MaxSize).toString()) - status = index.data(OverviewModel.Status).toString() - - def formatEta(seconds): #TODO add to utils - if seconds <= 0: return "" - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return _("ETA: ") + "%.2i:%.2i:%.2i" % (hours, minutes, seconds) - - statusline = QString(_("Parts: ") + "%s/%s" % (partsf, parts)) - if partsf == parts: - speedline = _("Finished") - elif not status == _("Downloading"): - speedline = QString(status) - else: - speedline = QString(formatEta(eta) + " " + _("Speed: %s") % formatSpeed(speed)) - - if progress in (0,100): - sizeline = QString(_("Size:") + "%s" % formatSize(maxSize)) - else: - sizeline = QString(_("Size:") + "%s / %s" % (formatSize(currentSize), formatSize(maxSize))) - - f = painter.font() - f.setPointSize(12) - f.setBold(True) - painter.setFont(f) - - r = option.rect.adjusted(4, 4, -4, -4) - painter.drawText(r.left(), r.top(), r.width(), r.height(), Qt.AlignTop | Qt.AlignLeft, packagename) - newr = painter.boundingRect(r.left(), r.top(), r.width(), r.height(), Qt.AlignTop | Qt.AlignLeft, packagename) - - f.setPointSize(10) - f.setBold(False) - painter.setFont(f) - - painter.drawText(r.left(), newr.bottom()+5, r.width(), r.height(), Qt.AlignTop | Qt.AlignLeft, statusline) - painter.drawText(r.left(), newr.bottom()+5, r.width(), r.height(), Qt.AlignTop | Qt.AlignHCenter, sizeline) - painter.drawText(r.left(), newr.bottom()+5, r.width(), r.height(), Qt.AlignTop | Qt.AlignRight, speedline) - newr = painter.boundingRect(r.left(), newr.bottom()+2, r.width(), r.height(), Qt.AlignTop | Qt.AlignLeft, statusline) - newr.setTop(newr.bottom()+8) - newr.setBottom(newr.top()+20) - newr.setRight(self.parent.width()-25) - - f.setPointSize(10) - painter.setFont(f) - - opts = QStyleOptionProgressBarV2() - opts.maximum = 100 - opts.minimum = 0 - opts.progress = progress - opts.rect = newr - opts.textVisible = True - opts.textAlignment = Qt.AlignCenter - opts.text = QString.number(opts.progress) + "%" - QApplication.style().drawControl(QStyle.CE_ProgressBar, opts, painter) - - def sizeHint(self, option, index): - return QSize(self.parent.width()-22, 59+16) diff --git a/module/gui/PackageDock.py b/module/gui/PackageDock.py deleted file mode 100644 index 73db8f177..000000000 --- a/module/gui/PackageDock.py +++ /dev/null @@ -1,90 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" -import re - -from PyQt4.QtCore import * -from PyQt4.QtGui import * - -class NewPackageDock(QDockWidget): - def __init__(self): - QDockWidget.__init__(self, _("New Package")) - self.setObjectName("New Package Dock") - self.widget = NewPackageWindow(self) - self.setWidget(self.widget) - self.setAllowedAreas(Qt.RightDockWidgetArea|Qt.LeftDockWidgetArea) - self.hide() - - def slotDone(self): - text = str(self.widget.box.toPlainText()) - pw = str(self.widget.passwordInput.text()) - if not pw: - pw = None - lines = [] - for line in text.splitlines(): - line = line.strip() - if not line: - continue - lines.append(line) - self.emit(SIGNAL("done"), str(self.widget.nameInput.text()), lines, pw) - self.widget.nameInput.setText("") - self.widget.passwordInput.setText("") - self.widget.box.clear() - self.hide() - - def parseUri(self): - - text=str(self.widget.box.toPlainText()) - self.widget.box.setText("") - result = re.findall(r"(?:ht|f)tps?:\/\/[a-zA-Z0-9\-\.\/\?=_&%#]+[<| |\"|\'|\r|\n|\t]{1}", text) - for url in result: - if "\n" or "\t" or "\r" or "\"" or "<" or "'" in url: - url = url[:-1] - self.widget.box.append("%s " % url) - -class NewPackageWindow(QWidget): - def __init__(self, dock): - QWidget.__init__(self) - self.dock = dock - self.setLayout(QGridLayout()) - layout = self.layout() - - nameLabel = QLabel(_("Name")) - nameInput = QLineEdit() - passwordLabel = QLabel(_("Password")) - passwordInput = QLineEdit() - - linksLabel = QLabel(_("Links in this Package")) - - self.box = QTextEdit() - self.nameInput = nameInput - self.passwordInput = passwordInput - - save = QPushButton(_("Create")) - parseUri = QPushButton(_("Filter URLs")) - - layout.addWidget(nameLabel, 0, 0) - layout.addWidget(nameInput, 0, 1) - layout.addWidget(passwordLabel, 1, 0) - layout.addWidget(passwordInput, 1, 1) - layout.addWidget(linksLabel, 2, 0, 1, 2) - layout.addWidget(self.box, 3, 0, 1, 2) - layout.addWidget(parseUri, 4, 0, 1, 2) - layout.addWidget(save, 5, 0, 1, 2) - - self.connect(save, SIGNAL("clicked()"), self.dock.slotDone) - self.connect(parseUri, SIGNAL("clicked()"), self.dock.parseUri)
\ No newline at end of file diff --git a/module/gui/Queue.py b/module/gui/Queue.py deleted file mode 100644 index 0a0cbb810..000000000 --- a/module/gui/Queue.py +++ /dev/null @@ -1,390 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from PyQt4.QtCore import * -from PyQt4.QtGui import * - -from time import time - -from module.remote.thriftbackend.ThriftClient import Destination -from module.gui.Collector import CollectorModel, Package, Link, CollectorView, statusMapReverse -from module.utils import formatSize, formatSpeed - -class QueueModel(CollectorModel): - """ - model for the queue view, inherits from CollectorModel - """ - - def __init__(self, view, connector): - CollectorModel.__init__(self, view, connector) - self.cols = 6 - self.wait_dict = {} - - self.updater = self.QueueUpdater(self.interval) - self.connect(self.updater, SIGNAL("update()"), self.update) - - class QueueUpdater(QObject): - """ - timer which emits signal for a download status reload - @TODO: make intervall configurable - """ - - def __init__(self, interval): - QObject.__init__(self) - - self.interval = interval - self.timer = QTimer() - self.timer.connect(self.timer, SIGNAL("timeout()"), self, SIGNAL("update()")) - - def start(self): - self.timer.start(1000) - - def stop(self): - self.timer.stop() - - def start(self): - self.updater.start() - - def stop(self): - self.updater.stop() - - def fullReload(self): - """ - reimplements CollectorModel.fullReload, because we want the Queue data - """ - self._data = [] - order = self.connector.getPackageOrder(Destination.Queue) - self.beginInsertRows(QModelIndex(), 0, len(order.values())) - for position, pid in order.iteritems(): - pack = self.connector.getPackageData(pid) - package = Package(pack) - self._data.append(package) - self._data = sorted(self._data, key=lambda p: p.data["order"]) - self.endInsertRows() - self.updateCount() - - def insertEvent(self, event): - """ - wrap CollectorModel.insertEvent to update the element count - """ - CollectorModel.insertEvent(self, event) - self.updateCount() - - def removeEvent(self, event): - """ - wrap CollectorModel.removeEvent to update the element count - """ - CollectorModel.removeEvent(self, event) - self.updateCount() - - def updateEvent(self, event): - """ - wrap CollectorModel.updateEvent to update the element count - """ - CollectorModel.updateEvent(self, event) - self.updateCount() - - def updateCount(self): - """ - calculate package- and filecount for statusbar, - ugly?: Overview connects to this signal for updating - """ - packageCount = len(self._data) - fileCount = 0 - for p in self._data: - fileCount += len(p.children) - self.mutex.unlock() - self.emit(SIGNAL("updateCount"), packageCount, fileCount) - self.mutex.lock() - - def update(self): - """ - update slot for download status updating - """ - locker = QMutexLocker(self.mutex) - downloading = self.connector.statusDownloads() - if not downloading: - return - for p, pack in enumerate(self._data): - for d in downloading: - child = pack.getChild(d.fid) - if child: - dd = { - "name": d.name, - "speed": d.speed, - "eta": d.eta, - "format_eta": d.format_eta, - "bleft": d.bleft, - "size": d.size, - "format_size": d.format_size, - "percent": d.percent, - "status": d.status, - "statusmsg": d.statusmsg, - "format_wait": d.format_wait, - "wait_until": d.wait_until - } - child.data["downloading"] = dd - k = pack.getChildKey(d.fid) - self.emit(SIGNAL("dataChanged(const QModelIndex &, const QModelIndex &)"), self.index(k, 0, self.index(p, 0)), self.index(k, self.cols, self.index(p, self.cols))) - self.updateCount() - - def headerData(self, section, orientation, role=Qt.DisplayRole): - """ - returns column heading - """ - if orientation == Qt.Horizontal and role == Qt.DisplayRole: - if section == 0: - return QVariant(_("Name")) - elif section == 2: - return QVariant(_("Status")) - elif section == 1: - return QVariant(_("Plugin")) - elif section == 3: - return QVariant(_("Size")) - elif section == 4: - return QVariant(_("ETA")) - elif section == 5: - return QVariant(_("Progress")) - return QVariant() - - def getWaitingProgress(self, item): - """ - returns time to wait, caches startingtime to provide progress - """ - locker = QMutexLocker(self.mutex) - if isinstance(item, Link): - if item.data["status"] == 5 and item.data["downloading"]: - until = float(item.data["downloading"]["wait_until"]) - try: - since, until_old = self.wait_dict[item.id] - if not until == until_old: - raise Exception - except: - since = time() - self.wait_dict[item.id] = since, until - since = float(since) - max_wait = float(until-since) - rest = int(until-time()) - if rest < 0: - return 0, None - res = 100/max_wait - perc = rest*res - return perc, rest - return None - - def getProgress(self, item, locked=True): - """ - return download progress, locks by default - since it's used in already locked calls, - it provides an option to not lock - """ - if locked: - locker = QMutexLocker(self.mutex) - if isinstance(item, Link): - try: - if item.data["status"] == 0: - return 100 - return int(item.data["downloading"]["percent"]) - except: - return 0 - elif isinstance(item, Package): - count = len(item.children) - perc_sum = 0 - for child in item.children: - try: - if child.data["status"] == 0: #completed - perc_sum += 100 - perc_sum += int(child.data["downloading"]["percent"]) - except: - pass - if count == 0: - return 0 - return perc_sum/count - return 0 - - def getSpeed(self, item): - """ - calculate download speed - """ - if isinstance(item, Link): - if item.data["downloading"]: - return int(item.data["downloading"]["speed"]) - elif isinstance(item, Package): - count = len(item.children) - speed_sum = 0 - all_waiting = True - running = False - for child in item.children: - val = 0 - if child.data["downloading"]: - if not child.data["statusmsg"] == "waiting": - all_waiting = False - val = int(child.data["downloading"]["speed"]) - running = True - speed_sum += val - if count == 0 or not running or all_waiting: - return None - return speed_sum - return None - - def data(self, index, role=Qt.DisplayRole): - """ - return cell data - """ - if not index.isValid(): - return QVariant() - if role == Qt.DisplayRole: - if index.column() == 0: - return QVariant(index.internalPointer().data["name"]) - elif index.column() == 1: - item = index.internalPointer() - plugins = [] - if isinstance(item, Package): - for child in item.children: - if not child.data["plugin"] in plugins: - plugins.append(child.data["plugin"]) - else: - plugins.append(item.data["plugin"]) - return QVariant(", ".join(plugins)) - elif index.column() == 2: - item = index.internalPointer() - status = 0 - speed = self.getSpeed(item) - if isinstance(item, Package): - for child in item.children: - if child.data["status"] > status: - status = child.data["status"] - else: - status = item.data["status"] - - if speed is None or status == 7 or status == 10 or status == 5: - return QVariant(self.translateStatus(statusMapReverse[status])) - else: - return QVariant("%s (%s)" % (self.translateStatus(statusMapReverse[status]), formatSpeed(speed))) - elif index.column() == 3: - item = index.internalPointer() - if isinstance(item, Link): - if item.data["status"] == 0: #TODO needs change?? - #self.getProgress(item, False) == 100: - return QVariant(formatSize(item.data["size"])) - elif self.getProgress(item, False) == 0: - try: - return QVariant("%s / %s" % (formatSize(item.data["size"]-item.data["downloading"]["bleft"]), formatSize(item.data["size"]))) - except: - return QVariant("0 B / %s" % formatSize(item.data["size"])) - else: - try: - return QVariant("%s / %s" % (formatSize(item.data["size"]-item.data["downloading"]["bleft"]), formatSize(item.data["size"]))) - except: - return QVariant("? / %s" % formatSize(item.data["size"])) - else: - ms = 0 - cs = 0 - for c in item.children: - try: - s = c.data["downloading"]["size"] - except: - s = c.data["size"] - if c.data["downloading"]: - cs += s - c.data["downloading"]["bleft"] - elif self.getProgress(c, False) == 100: - cs += s - ms += s - if cs == 0 or cs == ms: - return QVariant(formatSize(ms)) - else: - return QVariant("%s / %s" % (formatSize(cs), formatSize(ms))) - elif index.column() == 4: - item = index.internalPointer() - if isinstance(item, Link): - if item.data["downloading"]: - return QVariant(item.data["downloading"]["format_eta"]) - elif role == Qt.EditRole: - if index.column() == 0: - return QVariant(index.internalPointer().data["name"]) - return QVariant() - - def flags(self, index): - """ - cell flags - """ - if index.column() == 0 and self.parent(index) == QModelIndex(): - return Qt.ItemIsSelectable | Qt.ItemIsEditable | Qt.ItemIsEnabled - return Qt.ItemIsSelectable | Qt.ItemIsEnabled - -class QueueView(CollectorView): - """ - view component for queue - """ - - def __init__(self, connector): - CollectorView.__init__(self, connector) - self.setModel(QueueModel(self, connector)) - - self.setColumnWidth(0, 300) - self.setColumnWidth(1, 100) - self.setColumnWidth(2, 140) - self.setColumnWidth(3, 180) - self.setColumnWidth(4, 70) - - self.setEditTriggers(QAbstractItemView.NoEditTriggers) - - self.delegate = QueueProgressBarDelegate(self, self.model()) - self.setItemDelegateForColumn(5, self.delegate) - -class QueueProgressBarDelegate(QItemDelegate): - """ - used to display a progressbar in the progress cell - """ - - def __init__(self, parent, queue): - QItemDelegate.__init__(self, parent) - self.queue = queue - - def paint(self, painter, option, index): - """ - paint the progressbar - """ - if not index.isValid(): - return - if index.column() == 5: - item = index.internalPointer() - w = self.queue.getWaitingProgress(item) - wait = None - if w: - progress = w[0] - wait = w[1] - else: - progress = self.queue.getProgress(item) - opts = QStyleOptionProgressBarV2() - opts.maximum = 100 - opts.minimum = 0 - opts.progress = progress - opts.rect = option.rect - opts.rect.setRight(option.rect.right()-1) - opts.rect.setHeight(option.rect.height()-1) - opts.textVisible = True - opts.textAlignment = Qt.AlignCenter - if not wait is None: - opts.text = QString(_("waiting %d seconds") % (wait,)) - else: - opts.text = QString.number(opts.progress) + "%" - QApplication.style().drawControl(QStyle.CE_ProgressBar, opts, painter) - return - QItemDelegate.paint(self, painter, option, index) - diff --git a/module/gui/SettingsWidget.py b/module/gui/SettingsWidget.py deleted file mode 100644 index cd22a7b9e..000000000 --- a/module/gui/SettingsWidget.py +++ /dev/null @@ -1,202 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from PyQt4.QtCore import * -from PyQt4.QtGui import * -from sip import delete - - -class SettingsWidget(QWidget): - def __init__(self): - QWidget.__init__(self) - self.connector = None - self.sections = {} - self.psections = {} - self.data = None - self.pdata = None - self.setSizePolicy(QSizePolicy.Ignored, QSizePolicy.Ignored) - - def setConnector(self, connector): - self.connector = connector - - def loadConfig(self): - if self.sections and self.psections: - self.data = self.connector.getConfig() - self.pdata = self.connector.getPluginConfig() - - self.reloadSection(self.sections, self.data) - self.reloadSection(self.psections, self.pdata) - - return - - if self.layout(): - delete(self.layout()) - - for s in self.sections.values()+self.psections.values(): - delete(s) - - self.sections = {} - self.setLayout(QVBoxLayout()) - self.clearConfig() - layout = self.layout() - layout.setSizeConstraint(QLayout.SetMinAndMaxSize) - - general = QTabWidget() - self.general = general - - plugins = QTabWidget() - self.plugins = plugins - - tab = QTabWidget() - self.tab = tab - - gw = QWidget() - gw.setLayout(QVBoxLayout()) - gw.layout().addWidget(self.general) - pw = QWidget() - pw.setLayout(QVBoxLayout()) - pw.layout().addWidget(self.plugins) - tab.addTab(gw, _("General")) - tab.addTab(pw, _("Plugins")) - - layout.addWidget(tab) - - self.data = self.connector.getConfig() - self.pdata = self.connector.getPluginConfig() - for k, section in self.data.iteritems(): - s = Section(section, general) - self.sections[k] = s - - for k, section in self.pdata.iteritems(): - s = Section(section, plugins, "plugin") - self.psections[k] = s - - rel = QPushButton(_("Reload")) - save = QPushButton(_("Save")) - - layout.addWidget(save) - - cont = QHBoxLayout() - cont.addWidget(rel) - cont.addWidget(save) - - layout.addLayout(cont) - - self.connect(save, SIGNAL("clicked()"), self.saveConfig) - self.connect(rel, SIGNAL("clicked()"), self.loadConfig) - - def clearConfig(self): - self.sections = {} - - def reloadSection(self, sections, pdata): - - for k, section in enumerate(pdata): - if k in sections: - widget = sections[k] - for item in section.items: - if item.name in widget.inputs: - i = widget.inputs[item.name] - - if item.type == "int": - i.setValue(int(item.value)) - elif not item.type.find(";") == -1: - i.setCurrentIndex(i.findText(item.value)) - elif item.type == "bool": - if True if item.value.lower() in ("1","true", "on", "an","yes") else False: - i.setCurrentIndex(0) - else: - i.setCurrentIndex(1) - else: - i.setText(item.value) - - - def saveConfig(self): - self.data = self.connector.getConfig() - self.pdata = self.connector.getPluginConfig() - - self.saveSection(self.sections, self.data) - self.saveSection(self.psections, self.pdata, "plugin") - - - def saveSection(self, sections, pdata, sec="core"): - for k, section in enumerate(pdata): - if k in sections: - widget = sections[k] - for item in section.items: - if item.name in widget.inputs: - i = widget.inputs[item.name] - - #TODO : unresolved reference: option - - if item.type == "int": - if i.value() != int(item.value): - self.connector.setConfigValue(k, option, i.value(), sec) - elif not item.type.find(";") == -1: - if i.currentText() != item.value: - self.connector.setConfigValue(k, option, i.currentText(), sec) - elif item.type == "bool": - if (True if item.value.lower() in ("1","true", "on", "an","yes") else False) ^ (not i.currentIndex()): - self.connector.setConfigValue(k, option, not i.currentIndex(), sec) - else: - if i.text() != item.value: - self.connector.setConfigValue(k, option, str(i.text()), sec) - -class Section(QGroupBox): - def __init__(self, data, parent, ctype="core"): - self.data = data - QGroupBox.__init__(self, data.description, parent) - self.labels = {} - self.inputs = {} - self.ctype = ctype - layout = QFormLayout(self) - self.setLayout(layout) - - sw = QWidget() - sw.setLayout(QVBoxLayout()) - sw.layout().addWidget(self) - - sa = QScrollArea() - sa.setWidgetResizable(True) - sa.setWidget(sw) - sa.setFrameShape(sa.NoFrame) - - parent.addTab(sa, data.description) - - for option in self.data.items: - if option.type == "int": - i = QSpinBox(self) - i.setMaximum(999999) - i.setValue(int(option.value)) - elif not option.type.find(";") == -1: - choices = option.type.split(";") - i = QComboBox(self) - i.addItems(choices) - i.setCurrentIndex(i.findText(option.value)) - elif option.type == "bool": - i = QComboBox(self) - i.addItem(_("Yes"), QVariant(True)) - i.addItem(_("No"), QVariant(False)) - if True if option.value.lower() in ("1","true", "on", "an","yes") else False: - i.setCurrentIndex(0) - else: - i.setCurrentIndex(1) - else: - i = QLineEdit(self) - i.setText(option.value) - layout.addRow(option.description, i) - layout.setFieldGrowthPolicy(QFormLayout.AllNonFixedFieldsGrow) diff --git a/module/gui/XMLParser.py b/module/gui/XMLParser.py deleted file mode 100644 index 5e3b7bf65..000000000 --- a/module/gui/XMLParser.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" -from __future__ import with_statement - -from PyQt4.QtCore import * -from PyQt4.QtGui import * -from PyQt4.QtXml import * - -import os - -class XMLParser(): - def __init__(self, data, dfile=""): - self.mutex = QMutex() - self.mutex.lock() - self.xml = QDomDocument() - self.file = data - self.dfile = dfile - self.mutex.unlock() - self.loadData() - self.root = self.xml.documentElement() - - def loadData(self): - self.mutex.lock() - f = self.file - if not os.path.exists(f): - f = self.dfile - with open(f, 'r') as fh: - content = fh.read() - self.xml.setContent(content) - self.mutex.unlock() - - def saveData(self): - self.mutex.lock() - content = self.xml.toString() - with open(self.file, 'w') as fh: - fh.write(content) - self.mutex.unlock() - return content - - def parseNode(self, node, ret_type="list"): - if ret_type == "dict": - childNodes = {} - else: - childNodes = [] - child = node.firstChild() - while True: - n = child.toElement() - if n.isNull(): - break - else: - if ret_type == "dict": - childNodes[str(n.tagName())] = n - else: - childNodes.append(n) - child = child.nextSibling() - return childNodes diff --git a/module/gui/__init__.py b/module/gui/__init__.py deleted file mode 100644 index 8d1c8b69c..000000000 --- a/module/gui/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/module/gui/connector.py b/module/gui/connector.py deleted file mode 100644 index c16ccd08e..000000000 --- a/module/gui/connector.py +++ /dev/null @@ -1,165 +0,0 @@ -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -SERVER_VERSION = "0.4.9" - -from time import sleep -from uuid import uuid4 as uuid - -from PyQt4.QtCore import * -from PyQt4.QtGui import * - -import socket - -from module.remote.thriftbackend.ThriftClient import ThriftClient, WrongLogin, NoSSL, NoConnection -from thrift.Thrift import TException - -class Connector(QObject): - """ - manages the connection to the pyload core via thrift - """ - - firstAttempt = True - - def __init__(self): - QObject.__init__(self) - self.mutex = QMutex() - self.connectionID = None - self.host = None - self.port = None - self.user = None - self.password = None - self.ssl = None - self.running = True - self.internal = False - self.proxy = self.Dummy() - - def setConnectionData(self, host, port, user, password, ssl=False): - """ - set connection data for connection attempt, called from slotConnect - """ - self.host = host - self.port = port - self.user = user - self.password = password - self.ssl = ssl - - def connectProxy(self): - """ - initialize thrift rpc client, - check for ssl, check auth, - setup dispatcher, - connect error signals, - check server version - """ - if self.internal: return True - - err = None - try: - client = ThriftClient(self.host, self.port, self.user, self.password) - except WrongLogin: - err = _("bad login credentials") - except NoSSL: - err = _("no ssl support") - except NoConnection: - err = _("can't connect to host") - if err: - if not Connector.firstAttempt: - self.emit(SIGNAL("errorBox"), err) - Connector.firstAttempt = False - return False - - self.proxy = DispatchRPC(self.mutex, client) - self.connect(self.proxy, SIGNAL("connectionLost"), self, SIGNAL("connectionLost")) - - server_version = self.proxy.getServerVersion() - self.connectionID = uuid().hex - - if not server_version == SERVER_VERSION: - self.emit(SIGNAL("errorBox"), _("server is version %(new)s client accepts version %(current)s") % { "new": server_version, "current": SERVER_VERSION}) - return False - - return True - - def __getattr__(self, attr): - """ - redirect rpc calls to dispatcher - """ - return getattr(self.proxy, attr) - - class Dummy(object): - """ - dummy rpc proxy, to prevent errors - """ - def __nonzero__(self): - return False - - def __getattr__(self, attr): - def dummy(*args, **kwargs): - return None - return dummy - -class DispatchRPC(QObject): - """ - wraps the thrift client, to catch critical exceptions (connection lost) - adds thread safety - """ - - def __init__(self, mutex, server): - QObject.__init__(self) - self.mutex = mutex - self.server = server - - def __getattr__(self, attr): - """ - redirect and wrap call in Wrapper instance, locks dispatcher - """ - self.mutex.lock() - self.fname = attr - f = self.Wrapper(getattr(self.server, attr), self.mutex, self) - return f - - class Wrapper(object): - """ - represents a rpc call - """ - - def __init__(self, f, mutex, dispatcher): - self.f = f - self.mutex = mutex - self.dispatcher = dispatcher - - def __call__(self, *args, **kwargs): - """ - instance is called, rpc is executed - exceptions are processed - finally dispatcher is unlocked - """ - lost = False - try: - return self.f(*args, **kwargs) - except socket.error: #necessary? - lost = True - except TException: - lost = True - finally: - self.mutex.unlock() - if lost: - from traceback import print_exc - print_exc() - self.dispatcher.emit(SIGNAL("connectionLost")) diff --git a/module/CaptchaManager.py b/module/interaction/CaptchaManager.py index 02cd10a11..02cd10a11 100644 --- a/module/CaptchaManager.py +++ b/module/interaction/CaptchaManager.py diff --git a/module/interaction/EventManager.py b/module/interaction/EventManager.py new file mode 100644 index 000000000..38faa3c46 --- /dev/null +++ b/module/interaction/EventManager.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- + +from traceback import print_exc +from time import time + +class EventManager: + """ + Handles all Event related task, also stores an Event queue for clients, so they can retrieve them later. + + **Known Events:** + Most hook methods exists as events. These are some additional known events. + + ===================== ================ =========================================================== + Name Arguments Description + ===================== ================ =========================================================== + metaEvent eventName, *args Called for every event, with eventName and orginal args + downloadPreparing fid A download was just queued and will be prepared now. + downloadStarts fid A plugin will immediately starts the download afterwards. + linksAdded links, pid Someone just added links, you are able to modify the links. + allDownloadsProcessed Every link was handled, pyload would idle afterwards. + allDownloadsFinished Every download in queue is finished. + unrarFinished folder, fname An Unrar job finished + configChanged sec, opt, value The config was changed. + ===================== ================ =========================================================== + + | Notes: + | allDownloadsProcessed is *always* called before allDownloadsFinished. + | configChanged is *always* called before pluginConfigChanged. + """ + + CLIENT_EVENTS = ("packageUpdated", "packageInserted", "linkUpdated", "packageDeleted") + + def __init__(self, core): + self.core = core + self.log = core.log + + # uuid : list of events + self.clients = {} + self.events = {"metaEvent": []} + + def getEvents(self, uuid): + """ Get accumulated events for uuid since last call, this also registeres new client """ + if uuid not in self.clients: + self.clients[uuid] = Client() + return self.clients[uuid].get() + + def addEvent(self, event, func): + """Adds an event listener for event name""" + if event in self.events: + if func in self.events[event]: + self.log.debug("Function already registered %s" % func) + else: + self.events[event].append(func) + else: + self.events[event] = [func] + + def removeEvent(self, event, func): + """removes previously added event listener""" + if event in self.events: + self.events[event].remove(func) + + def dispatchEvent(self, event, *args): + """dispatches event with args""" + for f in self.events["metaEvent"]: + try: + f(event, *args) + except Exception, e: + self.log.warning("Error calling event handler %s: %s, %s, %s" + % ("metaEvent", f, args, str(e))) + if self.core.debug: + print_exc() + + if event in self.events: + for f in self.events[event]: + try: + f(*args) + except Exception, e: + self.log.warning("Error calling event handler %s: %s, %s, %s" + % (event, f, args, str(e))) + if self.core.debug: + print_exc() + + # append to client event queue + if event in self.CLIENT_EVENTS: + for uuid, client in self.clients.items(): + if client.delete(): + del self.clients[uuid] + else: + client.append(event, args) + + + def removeFromEvents(self, func): + """ Removes func from all known events """ + for name, events in self.events.iteritems(): + if func in events: + events.remove(func) + + + +class Client: + + # delete clients after this time + TIMEOUT = 60 * 60 + # max events, if this value is reached you should assume that older events were dropped + MAX = 30 + + def __init__(self): + self.lastActive = time() + self.events = [] + + def delete(self): + return self.lastActive + self.TIMEOUT < time() + + def append(self, event, args): + ev = (event, args) + if ev not in self.events: + self.events.insert(0, ev) + + del self.events[self.MAX:] + + + def get(self): + self.lastActive = time() + + events = self.events + self.events = [] + + return [(ev, [str(x) for x in args]) for ev, args in events]
\ No newline at end of file diff --git a/module/interaction/InteractionManager.py b/module/interaction/InteractionManager.py new file mode 100644 index 000000000..5ebcd1fcd --- /dev/null +++ b/module/interaction/InteractionManager.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" +from utils import lock +from traceback import print_exc +from threading import Lock + +class InteractionManager: + """ + Class that gives ability to interact with the user. + Arbitary task with predefined output and input type can be set off. + Asyncronous callbacks and default values keeps the ability to fallback if no user is present. + """ + def __init__(self, core): + self.lock = Lock() + self.core = core + self.tasks = [] #task store, for outgoing tasks only + + self.ids = 0 #only for internal purpose + + def work(self): + """Mainloop that gets the work done""" + + def newTask(self, img, format, file, result_type): + task = CaptchaTask(self.ids, img, format, file, result_type) + self.ids += 1 + return task + + @lock + def removeTask(self, task): + if task in self.tasks: + self.tasks.remove(task) + + @lock + def getTask(self): + for task in self.tasks: + if task.status in ("waiting", "shared-user"): + return task + + @lock + def getTaskByID(self, tid): + for task in self.tasks: + if task.id == str(tid): #task ids are strings + self.lock.release() + return task + + def handleCaptcha(self, task): + cli = self.core.isClientConnected() + + if cli: #client connected -> should solve the captcha + task.setWaiting(50) #wait 50 sec for response + + for plugin in self.core.hookManager.activePlugins(): + try: + plugin.newCaptchaTask(task) + except: + if self.core.debug: + print_exc() + + if task.handler or cli: #the captcha was handled + self.tasks.append(task) + return True + + task.error = _("No Client connected for captcha decrypting") + + return False + + +if __name__ == "__main__": + + it = InteractionTask()
\ No newline at end of file diff --git a/module/interaction/InteractionTask.py b/module/interaction/InteractionTask.py new file mode 100644 index 000000000..97cb16794 --- /dev/null +++ b/module/interaction/InteractionTask.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" + +from module.Api import InteractionTask as BaseInteractionTask +from module.Api import Input, Output + +#noinspection PyUnresolvedReferences +class InteractionTask(BaseInteractionTask): + """ + General Interaction Task extends ITask defined by thrift with additional fields and methods. + """ + #: Plugins can put needed data here + storage = None + #: Timestamp when task expires + waitUntil = 0 + #: Default data to be used, or True if preset should be used + default = None + #: The received result as string representation + result = None + #: List of registered handles + handler = None + #: Callback functions + callbacks = None + #: Error Message + error = None + #: Status string + status = None + + def __init__(self, *args, **kwargs): + BaseInteractionTask.__init__(self, *args, **kwargs) + + # additional internal attributes + self.storage = {} + self.default = [] + self.handler = [] + self.callbacks = [] + + +class CaptchaTask: + def __init__(self, id, img, format, file, result_type='textual'): + self.id = str(id) + self.captchaImg = img + self.captchaFormat = format + self.captchaFile = file + self.captchaResultType = result_type + self.handler = [] #the hook plugins that will take care of the solution + self.result = None + self.waitUntil = None + self.error = None #error message + + self.status = "init" + self.data = {} #handler can store data here + + def getCaptcha(self): + return self.captchaImg, self.captchaFormat, self.captchaResultType + + def setResult(self, text): + if self.isTextual(): + self.result = text + if self.isPositional(): + try: + parts = text.split(',') + self.result = (int(parts[0]), int(parts[1])) + except: + self.result = None + + def getResult(self): + try: + res = self.result.encode("utf8", "replace") + except: + res = self.result + + return res + + def getStatus(self): + return self.status + + def setWaiting(self, sec): + """ let the captcha wait secs for the solution """ + self.waitUntil = max(time() + sec, self.waitUntil) + self.status = "waiting" + + def isWaiting(self): + if self.result or self.error or time() > self.waitUntil: + return False + + return True + + def isTextual(self): + """ returns if text is written on the captcha """ + return self.captchaResultType == 'textual' + + def isPositional(self): + """ returns if user have to click a specific region on the captcha """ + return self.captchaResultType == 'positional' + + def setWatingForUser(self, exclusive): + if exclusive: + self.status = "user" + else: + self.status = "shared-user" + + def timedOut(self): + return time() > self.waitUntil + + def invalid(self): + """ indicates the captcha was not correct """ + [x.captchaInvalid(self) for x in self.handler] + + def correct(self): + [x.captchaCorrect(self) for x in self.handler] + + def __str__(self): + return "<CaptchaTask '%s'>" % self.id diff --git a/module/interaction/__init__.py b/module/interaction/__init__.py new file mode 100644 index 000000000..de6d13128 --- /dev/null +++ b/module/interaction/__init__.py @@ -0,0 +1,2 @@ +__author__ = 'christian' +
\ No newline at end of file diff --git a/module/lib/new_collections.py b/module/lib/new_collections.py new file mode 100644 index 000000000..12d05b4b9 --- /dev/null +++ b/module/lib/new_collections.py @@ -0,0 +1,375 @@ +## {{{ http://code.activestate.com/recipes/576693/ (r9) +# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. +# Passes Python2.7's test suite and incorporates all the latest updates. + +try: + from thread import get_ident as _get_ident +except ImportError: + from dummy_thread import get_ident as _get_ident + +try: + from _abcoll import KeysView, ValuesView, ItemsView +except ImportError: + pass + + +class OrderedDict(dict): + 'Dictionary that remembers insertion order' + # An inherited dict maps keys to values. + # The inherited dict provides __getitem__, __len__, __contains__, and get. + # The remaining methods are order-aware. + # Big-O running times for all methods are the same as for regular dictionaries. + + # The internal self.__map dictionary maps keys to links in a doubly linked list. + # The circular doubly linked list starts and ends with a sentinel element. + # The sentinel element never gets deleted (this simplifies the algorithm). + # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + + def __init__(self, *args, **kwds): + '''Initialize an ordered dictionary. Signature is the same as for + regular dictionaries, but keyword arguments are not recommended + because their insertion order is arbitrary. + + ''' + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__root + except AttributeError: + self.__root = root = [] # sentinel node + root[:] = [root, root, None] + self.__map = {} + self.__update(*args, **kwds) + + def __setitem__(self, key, value, dict_setitem=dict.__setitem__): + 'od.__setitem__(i, y) <==> od[i]=y' + # Setting a new item creates a new link which goes at the end of the linked + # list, and the inherited dictionary is updated with the new key/value pair. + if key not in self: + root = self.__root + last = root[0] + last[1] = root[0] = self.__map[key] = [last, root, key] + dict_setitem(self, key, value) + + def __delitem__(self, key, dict_delitem=dict.__delitem__): + 'od.__delitem__(y) <==> del od[y]' + # Deleting an existing item uses self.__map to find the link which is + # then removed by updating the links in the predecessor and successor nodes. + dict_delitem(self, key) + link_prev, link_next, key = self.__map.pop(key) + link_prev[1] = link_next + link_next[0] = link_prev + + def __iter__(self): + 'od.__iter__() <==> iter(od)' + root = self.__root + curr = root[1] + while curr is not root: + yield curr[2] + curr = curr[1] + + def __reversed__(self): + 'od.__reversed__() <==> reversed(od)' + root = self.__root + curr = root[0] + while curr is not root: + yield curr[2] + curr = curr[0] + + def clear(self): + 'od.clear() -> None. Remove all items from od.' + try: + for node in self.__map.itervalues(): + del node[:] + root = self.__root + root[:] = [root, root, None] + self.__map.clear() + except AttributeError: + pass + dict.clear(self) + + def popitem(self, last=True): + '''od.popitem() -> (k, v), return and remove a (key, value) pair. + Pairs are returned in LIFO order if last is true or FIFO order if false. + + ''' + if not self: + raise KeyError('dictionary is empty') + root = self.__root + if last: + link = root[0] + link_prev = link[0] + link_prev[1] = root + root[0] = link_prev + else: + link = root[1] + link_next = link[1] + root[1] = link_next + link_next[0] = root + key = link[2] + del self.__map[key] + value = dict.pop(self, key) + return key, value + + # -- the following methods do not depend on the internal structure -- + + def keys(self): + 'od.keys() -> list of keys in od' + return list(self) + + def values(self): + 'od.values() -> list of values in od' + return [self[key] for key in self] + + def items(self): + 'od.items() -> list of (key, value) pairs in od' + return [(key, self[key]) for key in self] + + def iterkeys(self): + 'od.iterkeys() -> an iterator over the keys in od' + return iter(self) + + def itervalues(self): + 'od.itervalues -> an iterator over the values in od' + for k in self: + yield self[k] + + def iteritems(self): + 'od.iteritems -> an iterator over the (key, value) items in od' + for k in self: + yield (k, self[k]) + + def update(*args, **kwds): + '''od.update(E, **F) -> None. Update od from dict/iterable E and F. + + If E is a dict instance, does: for k in E: od[k] = E[k] + If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] + Or if E is an iterable of items, does: for k, v in E: od[k] = v + In either case, this is followed by: for k, v in F.items(): od[k] = v + + ''' + if len(args) > 2: + raise TypeError('update() takes at most 2 positional ' + 'arguments (%d given)' % (len(args),)) + elif not args: + raise TypeError('update() takes at least 1 argument (0 given)') + self = args[0] + # Make progressively weaker assumptions about "other" + other = () + if len(args) == 2: + other = args[1] + if isinstance(other, dict): + for key in other: + self[key] = other[key] + elif hasattr(other, 'keys'): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + __update = update # let subclasses override update without breaking __init__ + + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + if default is self.__marker: + raise KeyError(key) + return default + + def setdefault(self, key, default=None): + 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' + if key in self: + return self[key] + self[key] = default + return default + + def __repr__(self, _repr_running={}): + 'od.__repr__() <==> repr(od)' + call_key = id(self), _get_ident() + if call_key in _repr_running: + return '...' + _repr_running[call_key] = 1 + try: + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + finally: + del _repr_running[call_key] + + def __reduce__(self): + 'Return state information for pickling' + items = [[k, self[k]] for k in self] + inst_dict = vars(self).copy() + for k in vars(OrderedDict()): + inst_dict.pop(k, None) + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S + and values equal to v (which defaults to None). + + ''' + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return len(self)==len(other) and self.items() == other.items() + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other + + # -- the following methods are only used in Python 2.7 -- + + def viewkeys(self): + "od.viewkeys() -> a set-like object providing a view on od's keys" + return KeysView(self) + + def viewvalues(self): + "od.viewvalues() -> an object providing a view on od's values" + return ValuesView(self) + + def viewitems(self): + "od.viewitems() -> a set-like object providing a view on od's items" + return ItemsView(self) +## end of http://code.activestate.com/recipes/576693/ }}} + +## {{{ http://code.activestate.com/recipes/500261/ (r15) +from operator import itemgetter as _itemgetter +from keyword import iskeyword as _iskeyword +import sys as _sys + +def namedtuple(typename, field_names, verbose=False, rename=False): + """Returns a new subclass of tuple with named fields. + + >>> Point = namedtuple('Point', 'x y') + >>> Point.__doc__ # docstring for the new class + 'Point(x, y)' + >>> p = Point(11, y=22) # instantiate with positional args or keywords + >>> p[0] + p[1] # indexable like a plain tuple + 33 + >>> x, y = p # unpack like a regular tuple + >>> x, y + (11, 22) + >>> p.x + p.y # fields also accessable by name + 33 + >>> d = p._asdict() # convert to a dictionary + >>> d['x'] + 11 + >>> Point(**d) # convert from a dictionary + Point(x=11, y=22) + >>> p._replace(x=100) # _replace() is like str.replace() but targets named fields + Point(x=100, y=22) + + """ + + # Parse and validate the field names. Validation serves two purposes, + # generating informative error messages and preventing template injection attacks. + if isinstance(field_names, basestring): + field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas + field_names = tuple(map(str, field_names)) + if rename: + names = list(field_names) + seen = set() + for i, name in enumerate(names): + if (not min(c.isalnum() or c=='_' for c in name) or _iskeyword(name) + or not name or name[0].isdigit() or name.startswith('_') + or name in seen): + names[i] = '_%d' % i + seen.add(name) + field_names = tuple(names) + for name in (typename,) + field_names: + if not min(c.isalnum() or c=='_' for c in name): + raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name) + if _iskeyword(name): + raise ValueError('Type names and field names cannot be a keyword: %r' % name) + if name[0].isdigit(): + raise ValueError('Type names and field names cannot start with a number: %r' % name) + seen_names = set() + for name in field_names: + if name.startswith('_') and not rename: + raise ValueError('Field names cannot start with an underscore: %r' % name) + if name in seen_names: + raise ValueError('Encountered duplicate field name: %r' % name) + seen_names.add(name) + + # Create and fill-in the class template + numfields = len(field_names) + argtxt = repr(field_names).replace("'", "")[1:-1] # tuple repr without parens or quotes + reprtxt = ', '.join('%s=%%r' % name for name in field_names) + template = '''class %(typename)s(tuple): + '%(typename)s(%(argtxt)s)' \n + __slots__ = () \n + _fields = %(field_names)r \n + def __new__(_cls, %(argtxt)s): + return _tuple.__new__(_cls, (%(argtxt)s)) \n + @classmethod + def _make(cls, iterable, new=tuple.__new__, len=len): + 'Make a new %(typename)s object from a sequence or iterable' + result = new(cls, iterable) + if len(result) != %(numfields)d: + raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result)) + return result \n + def __repr__(self): + return '%(typename)s(%(reprtxt)s)' %% self \n + def _asdict(self): + 'Return a new dict which maps field names to their values' + return dict(zip(self._fields, self)) \n + def _replace(_self, **kwds): + 'Return a new %(typename)s object replacing specified fields with new values' + result = _self._make(map(kwds.pop, %(field_names)r, _self)) + if kwds: + raise ValueError('Got unexpected field names: %%r' %% kwds.keys()) + return result \n + def __getnewargs__(self): + return tuple(self) \n\n''' % locals() + for i, name in enumerate(field_names): + template += ' %s = _property(_itemgetter(%d))\n' % (name, i) + if verbose: + print template + + # Execute the template string in a temporary namespace + namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, + _property=property, _tuple=tuple) + try: + exec template in namespace + except SyntaxError, e: + raise SyntaxError(e.message + ':\n' + template) + result = namespace[typename] + + # For pickling to work, the __module__ variable needs to be set to the frame + # where the named tuple is created. Bypass this step in enviroments where + # sys._getframe is not defined (Jython for example) or sys._getframe is not + # defined for arguments greater than 0 (IronPython). + try: + result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + + return result +## end of http://code.activestate.com/recipes/500261/ }}} diff --git a/module/network/Browser.py b/module/network/Browser.py index d68a23687..3452184d8 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -55,6 +55,13 @@ class Browser(object): return 0 @property + def name(self): + if self.dl: + return self.dl.name + else: + return "" + + @property def arrived(self): if self.dl: return self.dl.arrived diff --git a/module/network/Bucket.py b/module/network/Bucket.py index 69da277ae..ff80bda55 100644 --- a/module/network/Bucket.py +++ b/module/network/Bucket.py @@ -20,15 +20,18 @@ from time import time from threading import Lock +# 10kb minimum rate +MIN_RATE = 10240 + class Bucket: def __init__(self): - self.rate = 0 + self.rate = 0 # bytes per second, maximum targeted throughput self.tokens = 0 self.timestamp = time() self.lock = Lock() def __nonzero__(self): - return False if self.rate < 10240 else True + return False if self.rate < MIN_RATE else True def setRate(self, rate): self.lock.acquire() @@ -37,7 +40,7 @@ class Bucket: def consumed(self, amount): """ return time the process have to sleep, after consumed specified amount """ - if self.rate < 10240: return 0 #min. 10kb, may become unresponsive otherwise + if self.rate < MIN_RATE: return 0 #May become unresponsive otherwise self.lock.acquire() self.calc_tokens() @@ -47,7 +50,6 @@ class Bucket: time = -self.tokens/float(self.rate) else: time = 0 - self.lock.release() return time diff --git a/module/network/CookieJar.py b/module/network/CookieJar.py index c05812334..a020d6f9e 100644 --- a/module/network/CookieJar.py +++ b/module/network/CookieJar.py @@ -20,10 +20,12 @@ from time import time class CookieJar(): - def __init__(self, pluginname, account=None): + def __init__(self, pluginname): self.cookies = {} - self.plugin = pluginname - self.account = account + self.pluginname = pluginname + + def __repr__(self): + return ("<CookieJar plugin=%s>\n\t" % self.pluginname) + "\n\t".join(self.cookies.values()) def addCookies(self, clist): for c in clist: @@ -33,18 +35,18 @@ class CookieJar(): def getCookies(self): return self.cookies.values() - def parseCookie(self, name): + def getCookie(self, name): if name in self.cookies: return self.cookies[name].split("\t")[6] else: return None - def getCookie(self, name): - return self.parseCookie(name) + def setCookie(self, domain, name, value, path="/", exp=None): + if not exp: exp = time() + 3600 * 24 * 180 - def setCookie(self, domain, name, value, path="/", exp=time()+3600*24*180): - s = ".%s TRUE %s FALSE %s %s %s" % (domain, path, exp, name, value) + # dot makes it valid on all subdomains + s = ".%s TRUE %s FALSE %s %s %s" % (domain.strip("."), path, exp, name, value) self.cookies[name] = s def clear(self): - self.cookies = {} + self.cookies = {}
\ No newline at end of file diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index b637aef32..3380fb733 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -20,10 +20,13 @@ from os import remove, stat, fsync from os.path import exists from time import sleep from re import search -from module.utils import fs_encode + import codecs import pycurl +from module.utils import remove_chars +from module.utils.fs import fs_encode + from HTTPRequest import HTTPRequest class WrongFormat(Exception): @@ -256,11 +259,13 @@ class HTTPChunk(HTTPRequest): if line.startswith("accept-ranges") and "bytes" in line: self.p.chunkSupport = True - if line.startswith("content-disposition") and "filename=" in line: - name = orgline.partition("filename=")[2] - name = name.replace('"', "").replace("'", "").replace(";", "").strip() - self.p.nameDisposition = name - self.log.debug("Content-Disposition: %s" % name) + if "content-disposition" in line: + + m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", line) + if m: + name = remove_chars(m.groupdict()['name'], "\"';").strip() + self.p._name = name + self.log.debug("Content-Disposition: %s" % name) if not self.resume and line.startswith("content-length"): self.p.size = int(line.split(":")[1]) diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index fe8075539..59d38beee 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -17,9 +17,9 @@ @author: RaNaN """ -from os import remove, fsync +from os import remove from os.path import dirname -from time import sleep, time +from time import time from shutil import move from logging import getLogger @@ -28,8 +28,8 @@ import pycurl from HTTPChunk import ChunkInfo, HTTPChunk from HTTPRequest import BadHeader -from module.plugins.Plugin import Abort -from module.utils import save_join, fs_encode +from module.plugins.Hoster import Abort +from module.utils.fs import save_join, fs_encode class HTTPDownload(): """ loads a url http + ftp """ @@ -49,7 +49,7 @@ class HTTPDownload(): self.abort = False self.size = 0 - self.nameDisposition = None #will be parsed from content disposition + self._name = ""# will be parsed from content disposition self.chunks = [] @@ -87,6 +87,10 @@ class HTTPDownload(): if not self.size: return 0 return (self.arrived * 100) / self.size + @property + def name(self): + return self._name if self.disposition else "" + def _copyChunks(self): init = fs_encode(self.info.getChunkName(0)) #initial chunk name @@ -113,8 +117,8 @@ class HTTPDownload(): remove(fname) #remove chunk fo.close() - if self.nameDisposition and self.disposition: - self.filename = save_join(dirname(self.filename), self.nameDisposition) + if self.name: + self.filename = save_join(dirname(self.filename), self.name) move(init, fs_encode(self.filename)) self.info.remove() #remove info file @@ -144,8 +148,7 @@ class HTTPDownload(): finally: self.close() - if self.nameDisposition and self.disposition: return self.nameDisposition - return None + return self.name def _download(self, chunks, resume): if not resume: diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 40f18f2a5..2f084efb5 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -25,20 +25,21 @@ from httplib import responses from logging import getLogger from cStringIO import StringIO -from module.plugins.Plugin import Abort +from module.plugins.Hoster import Abort def myquote(url): - return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") + return quote(url.encode('utf8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") def myurlencode(data): - return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ - y.encode('utf_8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) + data = dict(data) + return urlencode(dict((x.encode('utf8') if isinstance(x, unicode) else x, \ + y.encode('utf8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) bad_headers = range(400, 404) + range(405, 418) + range(500, 506) class BadHeader(Exception): def __init__(self, code, content=""): - Exception.__init__(self, "Bad server response: %s %s" % (code, responses[int(code)])) + Exception.__init__(self, "Bad server response: %s %s" % (code, responses.get(int(code), "Unknown Header"))) self.code = code self.content = content @@ -192,11 +193,20 @@ class HTTPRequest(): if just_header: self.c.setopt(pycurl.FOLLOWLOCATION, 0) self.c.setopt(pycurl.NOBODY, 1) - self.c.perform() - rep = self.header - self.c.setopt(pycurl.FOLLOWLOCATION, 1) - self.c.setopt(pycurl.NOBODY, 0) + # overwrite HEAD request, we want a common request type + if post: + self.c.setopt(pycurl.CUSTOMREQUEST, "POST") + else: + self.c.setopt(pycurl.CUSTOMREQUEST, "GET") + + try: + self.c.perform() + rep = self.header + finally: + self.c.setopt(pycurl.FOLLOWLOCATION, 1) + self.c.setopt(pycurl.NOBODY, 0) + self.c.unsetopt(pycurl.CUSTOMREQUEST) else: self.c.perform() @@ -261,7 +271,7 @@ class HTTPRequest(): #TODO: html_unescape as default except LookupError: - self.log.debug("No Decoder foung for %s" % encoding) + self.log.debug("No Decoder found for %s" % encoding) except Exception: self.log.debug("Error when decoding string from %s." % encoding) diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 5b1528281..12fd66c95 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -24,34 +24,25 @@ from Bucket import Bucket from HTTPRequest import HTTPRequest from CookieJar import CookieJar -from XDCCRequest import XDCCRequest - class RequestFactory(): def __init__(self, core): self.lock = Lock() self.core = core self.bucket = Bucket() self.updateBucket() - self.cookiejars = {} + @property def iface(self): return self.core.config["download"]["interface"] - def getRequest(self, pluginName, account=None, type="HTTP"): - self.lock.acquire() - - if type == "XDCC": - return XDCCRequest(proxies=self.getProxies()) - + def getRequest(self, pluginName, cj=None): req = Browser(self.bucket, self.getOptions()) - if account: - cj = self.getCookieJar(pluginName, account) + if cj: req.setCookieJar(cj) else: req.setCookieJar(CookieJar(pluginName)) - self.lock.release() return req def getHTTPRequest(self, **kwargs): @@ -67,16 +58,12 @@ class RequestFactory(): rep = h.load(*args, **kwargs) finally: h.close() - - return rep - def getCookieJar(self, pluginName, account=None): - if (pluginName, account) in self.cookiejars: - return self.cookiejars[(pluginName, account)] + return rep - cj = CookieJar(pluginName, account) - self.cookiejars[(pluginName, account)] = cj - return cj + def openCookieJar(self, pluginname): + """Create new CookieJar""" + return CookieJar(pluginname) def getProxies(self): """ returns a proxy list for the request classes """ @@ -106,7 +93,7 @@ class RequestFactory(): def getOptions(self): """returns options needed for pycurl""" - return {"interface": self.iface(), + return {"interface": self.iface, "proxies": self.getProxies(), "ipv6": self.core.config["download"]["ipv6"]} diff --git a/module/plugins/Account.py b/module/plugins/Account.py index c147404e0..780a8ee69 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -1,145 +1,165 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from random import choice from time import time from traceback import print_exc from threading import RLock -from Plugin import Base -from module.utils import compare_time, parseFileSize, lock +from module.utils import compare_time, parseFileSize, lock, from_string +from module.Api import AccountInfo +from module.network.CookieJar import CookieJar + +from Base import Base class WrongPassword(Exception): pass - -class Account(Base): +#noinspection PyUnresolvedReferences +class Account(Base, AccountInfo): """ Base class for every Account plugin. Just overwrite `login` and cookies will be stored and account becomes accessible in\ - associated hoster plugin. Plugin should also provide `loadAccountInfo` + associated hoster plugin. Plugin should also provide `loadAccountInfo`. \ + A instance of this class is created for every entered account, it holds all \ + fields of AccountInfo ttype, and can be set easily at runtime. """ - __name__ = "Account" - __version__ = "0.2" - __type__ = "account" - __description__ = """Account Plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + + # Default values + valid = True + validuntil = None + trafficleft = None + maxtraffic = None + premium = True + activated = True #: after that time [in minutes] pyload will relogin the account login_timeout = 600 #: account data will be reloaded after this time info_threshold = 600 + # known options + known_opt = ("time", "limitDL") - def __init__(self, manager, accounts): + + def __init__(self, manager, loginname, password, options): Base.__init__(self, manager.core) + if "activated" in options: + activated = from_string(options["activated"], "bool") + else: + activated = Account.activated + + for opt in self.known_opt: + if opt not in options: + options[opt] = "" + + for opt in options.keys(): + if opt not in self.known_opt: + del options[opt] + + # default account attributes + AccountInfo.__init__(self, self.__name__, loginname, Account.valid, Account.validuntil, Account.trafficleft, + Account.maxtraffic, Account.premium, activated, options) + self.manager = manager - self.accounts = {} - self.infos = {} # cache for account information + self.lock = RLock() + self.timestamp = 0 + self.login_ts = 0 # timestamp for login + self.cj = CookieJar(self.__name__) + self.password = password + self.error = None - self.timestamps = {} - self.setAccounts(accounts) self.init() def init(self): pass - def login(self, user, data, req): + def login(self, req): """login into account, the cookies will be saved so user can be recognized - :param user: loginname - :param data: data dictionary :param req: `Request` instance """ - pass + raise NotImplemented + + def relogin(self): + """ Force a login. """ + req = self.getAccountRequest() + try: + return self._login(req) + finally: + req.close() @lock - def _login(self, user, data): + def _login(self, req): # set timestamp for login - self.timestamps[user] = time() - - req = self.getAccountRequest(user) + self.login_ts = time() + try: - self.login(user, data, req) + try: + self.login(req) + except TypeError: #TODO: temporary + self.logDebug("Deprecated .login(...) signature ommit user, data") + self.login(self.loginname, {"password": self.password}, req) + + + self.valid = True except WrongPassword: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": _("Wrong Password")}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname + , "msg": _("Wrong Password")}) + self.valid = False except Exception, e: self.logWarning( - _("Could not login with account %(user)s | %(msg)s") % {"user": user - , "msg": e}) - data["valid"] = False + _("Could not login with account %(user)s | %(msg)s") % {"user": self.loginname + , "msg": e}) + self.valid = False if self.core.debug: print_exc() - finally: - if req: req.close() - - def relogin(self, user): - req = self.getAccountRequest(user) - if req: - req.cj.clear() - req.close() - if user in self.infos: - del self.infos[user] #delete old information - self._login(user, self.accounts[user]) + return self.valid - def setAccounts(self, accounts): - self.accounts = accounts - for user, data in self.accounts.iteritems(): - self._login(user, data) - self.infos[user] = {} + def restoreDefaults(self): + self.validuntil = Account.validuntil + self.trafficleft = Account.trafficleft + self.maxtraffic = Account.maxtraffic + self.premium = Account.premium - def updateAccounts(self, user, password=None, options={}): + def update(self, password=None, options=None): """ updates account and return true if anything changed """ - if user in self.accounts: - self.accounts[user]["valid"] = True #do not remove or accounts will not login - if password: - self.accounts[user]["password"] = password - self.relogin(user) - return True - if options: - before = self.accounts[user]["options"] - self.accounts[user]["options"].update(options) - return self.accounts[user]["options"] != before - else: - self.accounts[user] = {"password": password, "options": options, "valid": True} - self._login(user, self.accounts[user]) + self.login_ts = 0 + self.valid = True #set valid so it will be retried to login + + if "activated" in options: + self.activated = from_string(options["avtivated"], "bool") + + if password: + self.password = password + self.relogin() return True + if options: + # remove unknown options + for opt in options.keys(): + if opt not in self.known_opt: + del options[opt] - def removeAccount(self, user): - if user in self.accounts: - del self.accounts[user] - if user in self.infos: - del self.infos[user] - if user in self.timestamps: - del self.timestamps[user] + before = self.options + self.options.update(options) + return self.options != before + + def getAccountRequest(self): + return self.core.requestFactory.getRequest(self.__name__, self.cj) + + def getDownloadSettings(self): + """ Can be overwritten to change download settings. Default is no chunkLimit, multiDL, resumeDownload + + :return: (chunkLimit, multiDL, resumeDownload) / (int,bool,bool) + """ + return -1, True, True @lock - def getAccountInfo(self, name, force=False): + def getAccountInfo(self, force=False): """retrieve account infos for an user, do **not** overwrite this method!\\ just use it to retrieve infos in hoster plugins. see `loadAccountInfo` @@ -147,113 +167,76 @@ class Account(Base): :param force: reloads cached account information :return: dictionary with information """ - data = Account.loadAccountInfo(self, name) - - if force or name not in self.infos: - self.logDebug("Get Account Info for %s" % name) - req = self.getAccountRequest(name) + if force or self.timestamp + self.info_threshold * 60 < time(): + # make sure to login + req = self.getAccountRequest() + self.checkLogin(req) + self.logDebug("Get Account Info for %s" % self.loginname) try: - infos = self.loadAccountInfo(name, req) - if not type(infos) == dict: - raise Exception("Wrong return format") + try: + infos = self.loadAccountInfo(req) + except TypeError: #TODO: temporary + self.logDebug("Deprecated .loadAccountInfo(...) signature, ommit user argument.") + infos = self.loadAccountInfo(self.loginname, req) except Exception, e: infos = {"error": str(e)} - - if req: req.close() + finally: + req.close() self.logDebug("Account Info: %s" % str(infos)) - - infos["timestamp"] = time() - self.infos[name] = infos - elif "timestamp" in self.infos[name] and self.infos[name][ - "timestamp"] + self.info_threshold * 60 < time(): - self.logDebug("Reached timeout for account data") - self.scheduleRefresh(name) - - data.update(self.infos[name]) - return data - - def isPremium(self, user): - info = self.getAccountInfo(user) - return info["premium"] - - def loadAccountInfo(self, name, req=None): - """this should be overwritten in account plugin,\ - and retrieving account information for user - - :param name: - :param req: `Request` instance + self.timestamp = time() + + self.restoreDefaults() # reset to initial state + if type(infos) == dict: # copy result from dict to class + for k, v in infos.iteritems(): + if hasattr(self, k): + setattr(self, k, v) + else: + self.logDebug("Unknown attribute %s=%s" % (k, v)) + + #TODO: remove user + def loadAccountInfo(self, req): + """ Overwrite this method and set account attributes within this method. + + :param user: Deprecated + :param req: Request instance :return: """ - return { - "validuntil": None, # -1 for unlimited - "login": name, - #"password": self.accounts[name]["password"], #@XXX: security - "options": self.accounts[name]["options"], - "valid": self.accounts[name]["valid"], - "trafficleft": None, # in kb, -1 for unlimited - "maxtraffic": None, - "premium": True, #useful for free accounts - "timestamp": 0, #time this info was retrieved - "type": self.__name__, - } - - def getAllAccounts(self, force=False): - return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()] - - def getAccountRequest(self, user=None): - if not user: - user, data = self.selectAccount() - if not user: - return None - - req = self.core.requestFactory.getRequest(self.__name__, user) - return req - - def getAccountCookies(self, user=None): - if not user: - user, data = self.selectAccount() - if not user: - return None - - cj = self.core.requestFactory.getCookieJar(self.__name__, user) - return cj - - def getAccountData(self, user): - return self.accounts[user] + pass - def selectAccount(self): - """ returns an valid account name and data""" - usable = [] - for user, data in self.accounts.iteritems(): - if not data["valid"]: continue + def getAccountCookies(self, user): + self.logDebug("Deprecated method .getAccountCookies -> use account.cj") + return self.cj - if "time" in data["options"] and data["options"]["time"]: - time_data = "" - try: - time_data = data["options"]["time"][0] - start, end = time_data.split("-") - if not compare_time(start.split(":"), end.split(":")): - continue - except: - self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + def getAccountData(self, user): + self.logDebug("Deprecated method .getAccountData -> use fields directly") + return {"password": self.password} - if user in self.infos: - if "validuntil" in self.infos[user]: - if self.infos[user]["validuntil"] > 0 and time() > self.infos[user]["validuntil"]: - continue - if "trafficleft" in self.infos[user]: - if self.infos[user]["trafficleft"] == 0: - continue + def isPremium(self, user=None): + if user: self.logDebug("Deprecated Argument user for .isPremium()", user) + return self.premium - usable.append((user, data)) + def isUsable(self): + """Check several contraints to determine if account should be used""" + if not self.valid or not self.activated: return False - if not usable: return None, None - return choice(usable) + if self.options["time"]: + time_data = "" + try: + time_data = self.options["time"] + start, end = time_data.split("-") + if not compare_time(start.split(":"), end.split(":")): + return False + except: + self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data) + + if 0 < self.validuntil < time(): + return False + if self.trafficleft is 0: # test explicity for 0 + return False - def canUse(self): - return False if self.selectAccount() == (None, None) else True + return True def parseTraffic(self, string): #returns kbyte return parseFileSize(string) / 1024 @@ -261,32 +244,36 @@ class Account(Base): def wrongPassword(self): raise WrongPassword - def empty(self, user): - if user in self.infos: - self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user) + def empty(self, user=None): + if user: self.logDebug("Deprecated argument user for .empty()", user) + + self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % self.login) - self.infos[user].update({"trafficleft": 0}) - self.scheduleRefresh(user, 30 * 60) + self.trafficleft = 0 + self.scheduleRefresh(30 * 60) def expired(self, user): if user in self.infos: self.logWarning(_("Account %s is expired, checking again in 1h") % user) - self.infos[user].update({"validuntil": time() - 1}) - self.scheduleRefresh(user, 60 * 60) + self.validuntil = time() - 1 + self.scheduleRefresh(60 * 60) - def scheduleRefresh(self, user, time=0, force=True): + def scheduleRefresh(self, time=0, force=True): """ add task to refresh account info to sheduler """ - self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time)) - self.core.scheduler.addJob(time, self.getAccountInfo, [user, force]) + self.logDebug("Scheduled Account refresh for %s in %s seconds." % (self.loginname, time)) + self.core.scheduler.addJob(time, self.getAccountInfo, [force]) @lock - def checkLogin(self, user): + def checkLogin(self, req): """ checks if user is still logged in """ - if user in self.timestamps: - if self.timestamps[user] + self.login_timeout * 60 < time(): - self.logDebug("Reached login timeout for %s" % user) - self.relogin(user) - return False + if self.login_ts + self.login_timeout * 60 < time(): + if self.login_ts: # seperate from fresh login to have better debug logs + self.logDebug("Reached login timeout for %s" % self.loginname) + else: + self.logDebug("Login with %s" % self.loginname) + + self._login(req) + return False return True diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py index fc521d36c..c610d10e0 100644 --- a/module/plugins/AccountManager.py +++ b/module/plugins/AccountManager.py @@ -17,169 +17,125 @@ @author: RaNaN """ -from os.path import exists -from shutil import copy - from threading import Lock +from random import choice -from module.PullEvents import AccountUpdateEvent -from module.utils import chmod, lock - -ACC_VERSION = 1 +from module.common.json_layer import json +from module.utils import lock -class AccountManager(): +class AccountManager: """manages all accounts""" - #---------------------------------------------------------------------- def __init__(self, core): """Constructor""" self.core = core self.lock = Lock() - self.initPlugins() - self.saveAccounts() # save to add categories to conf + self.loadAccounts() - def initPlugins(self): - self.accounts = {} # key = ( plugin ) - self.plugins = {} + def loadAccounts(self): + """loads all accounts available""" - self.initAccountPlugins() - self.loadAccounts() + self.accounts = {} + for plugin, loginname, activated, password, options in self.core.db.loadAccounts(): + # put into options as used in other context + options = json.loads(options) if options else {} + options["activated"] = activated - def getAccountPlugin(self, plugin): - """get account instance for plugin or None if anonymous""" - if plugin in self.accounts: - if plugin not in self.plugins: - self.plugins[plugin] = self.core.pluginManager.loadClass("accounts", plugin)(self, self.accounts[plugin]) + self.createAccount(plugin, loginname, password, options) + + return + + def iterAccounts(self): + """ yields login, account for all accounts""" + for name, data in self.accounts.iteritems(): + for login, account in data.iteritems(): + yield login, account - return self.plugins[plugin] - else: - return None - - def getAccountPlugins(self): - """ get all account instances""" - - plugins = [] - for plugin in self.accounts.keys(): - plugins.append(self.getAccountPlugin(plugin)) - - return plugins - #---------------------------------------------------------------------- - def loadAccounts(self): - """loads all accounts available""" - - if not exists("accounts.conf"): - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - - f = open("accounts.conf", "rb") - content = f.readlines() - version = content[0].split(":")[1].strip() if content else "" - f.close() - - if not version or int(version) < ACC_VERSION: - copy("accounts.conf", "accounts.backup") - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - self.core.log.warning(_("Account settings deleted, due to new config format.")) - return - - - - plugin = "" - name = "" - - for line in content[1:]: - line = line.strip() - - if not line: continue - if line.startswith("#"): continue - if line.startswith("version"): continue - - if line.endswith(":") and line.count(":") == 1: - plugin = line[:-1] - self.accounts[plugin] = {} - - elif line.startswith("@"): - try: - option = line[1:].split() - self.accounts[plugin][name]["options"][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) - except: - pass - - elif ":" in line: - name, sep, pw = line.partition(":") - self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} - #---------------------------------------------------------------------- def saveAccounts(self): """save all account information""" - - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION) + "\n") - - for plugin, accounts in self.accounts.iteritems(): - f.write("\n") - f.write(plugin+":\n") - - for name,data in accounts.iteritems(): - f.write("\n\t%s:%s\n" % (name,data["password"]) ) - if data["options"]: - for option, values in data["options"].iteritems(): - f.write("\t@%s %s\n" % (option, " ".join(values))) - - f.close() - chmod(f.name, 0600) - - - #---------------------------------------------------------------------- - def initAccountPlugins(self): - """init names""" - for name in self.core.pluginManager.getAccountPlugins(): - self.accounts[name] = {} - + + data = [] + for name, plugin in self.accounts.iteritems(): + data.extend([(name, acc.loginname, acc.activated, acc.password, json.dumps(acc.options)) for acc in + plugin.itervalues()]) + self.core.db.saveAccounts(data) + + def createAccount(self, plugin, loginname, password, options): + klass = self.core.pluginManager.loadClass("accounts", plugin) + if not klass: + self.core.log.warning(_("Unknown account plugin %s") % plugin) + return + + if plugin not in self.accounts: + self.accounts[plugin] = {} + + self.core.log.debug("Create account %s:%s" % (plugin, loginname)) + + self.accounts[plugin][loginname] = klass(self, loginname, password, options) + + + def getAccount(self, plugin, user): + return self.accounts[plugin].get(user, None) + @lock - def updateAccount(self, plugin , user, password=None, options={}): + def updateAccount(self, plugin, user, password=None, options={}): """add or update account""" - if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - updated = p.updateAccounts(user, password, options) - #since accounts is a ref in plugin self.accounts doesnt need to be updated here - + if plugin in self.accounts and user in self.accounts[plugin]: + acc = self.accounts[plugin][user] + updated = acc.update(password, options) + self.saveAccounts() - if updated: p.scheduleRefresh(user, force=False) - + if updated: acc.scheduleRefresh(force=True) + else: + self.createAccount(plugin, user, password, options) + self.saveAccounts() + + self.sendChange(plugin, user) + @lock def removeAccount(self, plugin, user): """remove account""" - + if plugin in self.accounts and user in self.accounts[plugin]: + del self.accounts[plugin][user] + self.core.db.removeAccount(plugin, user) + self.core.eventManager.dispatchEvent("accountDeleted", plugin, user) + else: + self.core.log.debug("Remove non existing account %s %s" % (plugin, user)) + + + @lock + def getAccountForPlugin(self, plugin): if plugin in self.accounts: - p = self.getAccountPlugin(plugin) - p.removeAccount(user) + accs = [x for x in self.accounts[plugin].values() if x.isUsable()] + if accs: return choice(accs) - self.saveAccounts() + return None @lock - def getAccountInfos(self, force=True, refresh=False): - data = {} + def getAllAccounts(self, refresh=False): + """ Return account info, refresh afterwards if needed + :param refresh: + :return: + """ if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - force = False - - for p in self.accounts.keys(): - if self.accounts[p]: - p = self.getAccountPlugin(p) - data[p.__name__] = p.getAllAccounts(force) - else: - data[p] = [] - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) - return data - - def sendChange(self): - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) + self.core.scheduler.addJob(0, self.core.accountManager.getAllAccounts) + + # load unavailable account info + for p_dict in self.accounts.itervalues(): + for acc in p_dict.itervalues(): + acc.getAccountInfo() + + return self.accounts + + def refreshAllAccounts(self): + """ Force a refresh of every account """ + for p in self.accounts.itervalues(): + for acc in p.itervalues(): + acc.getAccountInfo(True) + + def sendChange(self, plugin, name): + self.core.eventManager.dispatchEvent("accountUpdated", plugin, name)
\ No newline at end of file diff --git a/module/plugins/Base.py b/module/plugins/Base.py new file mode 100644 index 000000000..34074095e --- /dev/null +++ b/module/plugins/Base.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" + +import sys +from module.utils import decode +from module.utils.fs import exists, makedirs, join + +# TODO +# more attributes if needed +# get rid of catpcha & container plugins ?! (move to crypter & internals) +# adapt old plugins as needed + +class Fail(Exception): + """ raised when failed """ + +class Retry(Exception): + """ raised when start again from beginning """ + +class Base(object): + """ + The Base plugin class with all shared methods and every possible attribute for plugin definition. + """ + __version__ = "0.1" + #: Regexp pattern which will be matched for download/crypter plugins + __pattern__ = r"" + #: Config definition: list of (name, type, verbose_name, default_value) or + #: (name, type, verbose_name, short_description, default_value) + __config__ = list() + #: Short description, one liner + __description__ = "" + #: More detailed text + __long_description__ = """""" + #: List of needed modules + __dependencies__ = tuple() + #: Tags to categorize the plugin + __tags__ = tuple() + #: Base64 encoded .png icon, please don't use sizes above ~3KB + __icon__ = "" + #: Alternative, link to png icon + __icon_url__ = "" + #: Url with general information/support/discussion + __url__ = "" + #: Url to term of content, user is accepting these when using the plugin + __toc_url__ = "" + #: Url to service (to buy premium) for accounts + __ref_url__ = "" + + __author_name__ = tuple() + __author_mail__ = tuple() + + + def __init__(self, core): + self.__name__ = self.__class__.__name__ + + #: Core instance + self.core = core + #: logging instance + self.log = core.log + #: core config + self.config = core.config + #: :class:`EventManager` + self.evm = core.eventManager + #: :class:`InteractionManager` + self.im = core.interActionManager + + def logInfo(self, *args, **kwargs): + """ Print args to log at specific level + + :param args: Arbitary object which should be logged + :param kwargs: sep=(how to seperate arguments), default = " | " + """ + self._log("info", *args, **kwargs) + + def logWarning(self, *args, **kwargs): + self._log("warning", *args, **kwargs) + + def logError(self, *args, **kwargs): + self._log("error", *args, **kwargs) + + def logDebug(self, *args, **kwargs): + self._log("debug", *args, **kwargs) + + def _log(self, level, *args, **kwargs): + if "sep" in kwargs: + sep = "%s" % kwargs["sep"] + else: + sep = " | " + + + strings = [] + for obj in args: + if type(obj) == unicode: + strings.append(obj) + elif type(obj) == str: + strings.append(decode(obj)) + else: + strings.append(str(obj)) + + getattr(self.log, level)("%s: %s" % (self.__name__, sep.join(strings))) + + def setConfig(self, option, value): + """ Set config value for current plugin + + :param option: + :param value: + """ + self.core.config.set(self.__name__, option, value) + + def getConf(self, option): + """ see `getConfig` """ + return self.core.config.get(self.__name__, option) + + def getConfig(self, option): + """ Returns config value for current plugin + + :param option: + :return: + """ + return self.getConf(option) + + def setStorage(self, key, value): + """ Saves a value persistently to the database """ + self.core.db.setStorage(self.__name__, key, value) + + def store(self, key, value): + """ same as `setStorage` """ + self.core.db.setStorage(self.__name__, key, value) + + def getStorage(self, key=None, default=None): + """ Retrieves saved value or dict of all saved entries if key is None """ + if key is not None: + return self.core.db.getStorage(self.__name__, key) or default + return self.core.db.getStorage(self.__name__, key) + + def retrieve(self, *args, **kwargs): + """ same as `getStorage` """ + return self.getStorage(*args, **kwargs) + + def delStorage(self, key): + """ Delete entry in db """ + self.core.db.delStorage(self.__name__, key) + + def shell(self): + """ open ipython shell """ + if self.core.debug: + from IPython import embed + #noinspection PyUnresolvedReferences + sys.stdout = sys._stdout + embed() + + def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): + """Load content at url and returns it + + :param url: url as string + :param get: GET as dict + :param post: POST as dict, list or string + :param ref: Set HTTP_REFERER header + :param cookies: use saved cookies + :param just_header: if True only the header will be retrieved and returned as dict + :param decode: Wether to decode the output according to http header, should be True in most cases + :return: Loaded content + """ + if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") + + res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) + + if self.core.debug: + from inspect import currentframe + + frame = currentframe() + if not exists(join("tmp", self.__name__)): + makedirs(join("tmp", self.__name__)) + + f = open( + join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) + , "wb") + del frame # delete the frame or it wont be cleaned + + try: + tmp = res.encode("utf8") + except: + tmp = res + + f.write(tmp) + f.close() + + if just_header: + #parse header + header = {"code": self.req.code} + for line in res.splitlines(): + line = line.strip() + if not line or ":" not in line: continue + + key, none, value = line.partition(":") + key = key.lower().strip() + value = value.strip() + + if key in header: + if type(header[key]) == list: + header[key].append(value) + else: + header[key] = [header[key], value] + else: + header[key] = value + res = header + + return res + + def fail(self, reason): + """ fail and give reason """ + raise Fail(reason)
\ No newline at end of file diff --git a/module/plugins/Container.py b/module/plugins/Container.py deleted file mode 100644 index c233d3710..000000000 --- a/module/plugins/Container.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Crypter import Crypter - -from os.path import join, exists, basename -from os import remove -import re - -class Container(Crypter): - __name__ = "Container" - __version__ = "0.1" - __pattern__ = None - __type__ = "container" - __description__ = """Base container plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - - def preprocessing(self, thread): - """prepare""" - - self.setup() - self.thread = thread - - self.loadToDisk() - - self.decrypt(self.pyfile) - self.deleteTmp() - - self.createPackages() - - - def loadToDisk(self): - """loads container to disk if its stored remotely and overwrite url, - or check existent on several places at disk""" - - if self.pyfile.url.startswith("http"): - self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1] - content = self.load(self.pyfile.url) - self.pyfile.url = join(self.config["general"]["download_folder"], self.pyfile.name) - f = open(self.pyfile.url, "wb" ) - f.write(content) - f.close() - - else: - self.pyfile.name = basename(self.pyfile.url) - if not exists(self.pyfile.url): - if exists(join(pypath, self.pyfile.url)): - self.pyfile.url = join(pypath, self.pyfile.url) - else: - self.fail(_("File not exists.")) - - - def deleteTmp(self): - if self.pyfile.name.startswith("tmp_"): - remove(self.pyfile.url) - - diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py index d1549fe80..6079ae8f6 100644 --- a/module/plugins/Crypter.py +++ b/module/plugins/Crypter.py @@ -1,72 +1,259 @@ # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -from module.plugins.Plugin import Plugin - -class Crypter(Plugin): - __name__ = "Crypter" - __version__ = "0.1" - __pattern__ = None - __type__ = "container" - __description__ = """Base crypter plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") - - def __init__(self, pyfile): - Plugin.__init__(self, pyfile) - - #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder ) - self.packages = [] +from traceback import print_exc + +from module.Api import Destination +from module.common.packagetools import parseNames +from module.utils import to_list, has_method, uniqify +from module.utils.fs import exists, remove, fs_encode + +from Base import Base, Retry + +class Package: + """ Container that indicates new package should be created """ + def __init__(self, name, urls=None, dest=Destination.Queue): + self.name = name + self.urls = urls if urls else [] + self.dest = dest + + def addUrl(self, url): + self.urls.append(url) + + def __eq__(self, other): + return self.name == other.name and self.urls == other.urls + + def __repr__(self): + return u"<CrypterPackage name=%s, links=%s, dest=%s" % (self.name, self.urls, self.dest) + + def __hash__(self): + return hash(self.name) ^ hash(frozenset(self.urls)) ^ hash(self.dest) + +class PyFileMockup: + """ Legacy class needed by old crypter plugins """ + def __init__(self, url, pack): + self.url = url + self.name = url + self._package = pack + self.packageid = pack.id if pack else -1 + + def package(self): + return self._package + +class Crypter(Base): + """ + Base class for (de)crypter plugins. Overwrite decrypt* methods. + + How to use decrypt* methods: + + You have to overwrite at least one method of decryptURL, decryptURLs, decryptFile. - #: List of urls, pyLoad will generate packagenames + After decrypting and generating urls/packages you have to return the result. + Valid return Data is: + + :class:`Package` instance Crypter.Package + A **new** package will be created with the name and the urls of the object. + + List of urls and `Package` instances + All urls in the list will be added to the **current** package. For each `Package`\ + instance a new package will be created. + + """ + + #: Prefix to annotate that the submited string for decrypting is indeed file content + CONTENT_PREFIX = "filecontent:" + + @classmethod + def decrypt(cls, core, url_or_urls): + """Static method to decrypt, something. Can be used by other plugins. + To decrypt file content prefix the string with ``CONTENT_PREFIX `` as seen above. + + :param core: pyLoad `Core`, needed in decrypt context + :param url_or_urls: List of urls or single url/ file content + :return: List of decrypted urls, all package info removed + """ + urls = to_list(url_or_urls) + p = cls(core) + try: + result = p.processDecrypt(urls) + finally: + p.clean() + + ret = [] + + for url_or_pack in result: + if isinstance(url_or_pack, Package): #package + ret.extend(url_or_pack.urls) + else: # single url + ret.append(url_or_pack) + # eliminate duplicates + return uniqify(ret) + + def __init__(self, core, package=None, password=None): + Base.__init__(self, core) + self.req = core.requestFactory.getRequest(self.__name__) + + # Package the plugin was initialized for, dont use this, its not guaranteed to be set + self.package = package + #: Password supplied by user + self.password = password + #: Propose a renaming of the owner package + self.rename = None + + # For old style decrypter, do not use these ! + self.packages = [] self.urls = [] - - self.multiDL = True - self.limitDL = 0 - - - def preprocessing(self, thread): - """prepare""" - self.setup() - self.thread = thread - - self.decrypt(self.pyfile) - - self.createPackages() - - - def decrypt(self, pyfile): + self.pyfile = None + + self.init() + + def init(self): + """More init stuff if needed""" + + def setup(self): + """Called everytime before decrypting. A Crypter plugin will be most likly used for several jobs.""" + + def decryptURL(self, url): + """Decrypt a single url + + :param url: url to decrypt + :return: See :class:`Crypter` Documentation + """ + if url.startswith("http"): # basic method to redirect + return self.decryptFile(self.load(url)) + else: + self.fail(_("Not existing file or unsupported protocol")) + + def decryptURLs(self, urls): + """Decrypt a bunch of urls + + :param urls: list of urls + :return: See :class:`Crypter` Documentation + """ raise NotImplementedError - def createPackages(self): - """ create new packages from self.packages """ - for pack in self.packages: + def decryptFile(self, content): + """Decrypt file content - self.log.debug("Parsed package %(name)s with %(len)d links" % { "name" : pack[0], "len" : len(pack[1]) } ) - - links = [x.decode("utf-8") for x in pack[1]] - - pid = self.core.api.addPackage(pack[0], links, self.pyfile.package().queue) + :param content: content to decrypt as string + :return: See :class:`Crypter` Documentation + """ + raise NotImplementedError + + def generatePackages(self, urls): + """Generates :class:`Package` instances and names from urls. Usefull for many different links and no\ + given package name. + + :param urls: list of urls + :return: list of `Package` + """ + return [Package(name, purls) for name, purls in parseNames([(url,url) for url in urls]).iteritems()] + + def _decrypt(self, urls): + """ Internal method to select decrypting method + + :param urls: List of urls/content + :return: + """ + cls = self.__class__ + + # seperate local and remote files + content, urls = self.getLocalContent(urls) + + if has_method(cls, "decryptURLs"): + self.setup() + result = to_list(self.decryptURLs(urls)) + elif has_method(cls, "decryptURL"): + result = [] + for url in urls: + self.setup() + result.extend(to_list(self.decryptURL(url))) + elif has_method(cls, "decrypt"): + self.logDebug("Deprecated .decrypt() method in Crypter plugin") + result = [] + for url in urls: + self.pyfile = PyFileMockup(url, self.package) + self.setup() + self.decrypt(self.pyfile) + result.extend(self.convertPackages()) + else: + if not has_method(cls, "decryptFile") or urls: + self.logDebug("No suited decrypting method was overwritten in plugin") + result = [] + + if has_method(cls, "decryptFile"): + for f, c in content: + self.setup() + result.extend(to_list(self.decryptFile(c))) + try: + if f.startswith("tmp_"): remove(f) + except : + pass + + return result + + def processDecrypt(self, urls): + """Catches all exceptions in decrypt methods and return results + + :return: Decrypting results + """ + try: + return self._decrypt(urls) + except Exception: + if self.core.debug: + print_exc() + return [] + + def getLocalContent(self, urls): + """Load files from disk and seperate to file content and url list + + :param urls: + :return: list of (filename, content), remote urls + """ + content = [] + # do nothing if no decryptFile method + if hasattr(self.__class__, "decryptFile"): + remote = [] + for url in urls: + path = None + if url.startswith("http"): # skip urls directly + pass + elif url.startswith(self.CONTENT_PREFIX): + path = url + elif exists(url): + path = url + elif exists(self.core.path(url)): + path = self.core.path(url) + + if path: + try: + if path.startswith(self.CONTENT_PREFIX): + content.append(("", path[len(self.CONTENT_PREFIX)])) + else: + f = open(fs_encode(path), "rb") + content.append((f.name, f.read())) + f.close() + except IOError, e: + self.logError("IOError", e) + else: + remote.append(url) + + #swap filtered url list + urls = remote + + return content, urls - if self.pyfile.package().password: - self.core.api.setPackageData(pid, {"password": self.pyfile.package().password}) + def retry(self): + """ Retry decrypting, will only work once. Somewhat deprecated method, should be avoided. """ + raise Retry() - if self.urls: - self.core.api.generateAndAddPackages(self.urls) + def convertPackages(self): + """ Deprecated """ + self.logDebug("Deprecated method .convertPackages()") + res = [Package(name, urls) for name, urls in self.packages] + res.extend(self.urls) + return res + def clean(self): + if hasattr(self, "req"): + self.req.close() + del self.req
\ No newline at end of file diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index 5efd08bae..83ef091ae 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -14,38 +14,50 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - @author: mkaay - @interface-version: 0.2 + @author: RaNaN """ from traceback import print_exc -from Plugin import Base +#from functools import wraps +from module.utils import has_method + +from Base import Base + +def class_name(p): + return p.rpartition(".")[2] class Expose(object): """ used for decoration to declare rpc services """ + def __new__(cls, f, *args, **kwargs): + hookManager.addRPC(class_name(f.__module__), f.func_name, f.func_doc) + return f + +def AddEventListener(event): + """ used to register method for events """ + class _klass(object): + def __new__(cls, f, *args, **kwargs): + hookManager.addEventListener(class_name(f.__module__), f.func_name, event) + return f + return _klass + +class ConfigHandler(object): + """ register method as config handler """ def __new__(cls, f, *args, **kwargs): - hookManager.addRPC(f.__module__, f.func_name, f.func_doc) + hookManager.addConfigHandler(class_name(f.__module__), f.func_name) return f def threaded(f): + #@wraps(f) def run(*args,**kwargs): hookManager.startThread(f, *args, **kwargs) return run class Hook(Base): """ - Base class for hook plugins. + Base class for hook plugins. Please use @threaded decorator for all longer running task. """ - __name__ = "Hook" - __version__ = "0.2" - __type__ = "hook" - __threaded__ = [] - __config__ = [ ("name", "type", "desc" , "default") ] - __description__ = """interface for hook""" - __author_name__ = ("mkaay", "RaNaN") - __author_mail__ = ("mkaay@mkaay.de", "RaNaN@pyload.org") #: automatically register event listeners for functions, attribute will be deleted dont use it yourself event_map = None @@ -75,20 +87,21 @@ class Hook(Base): for event, funcs in self.event_map.iteritems(): if type(funcs) in (list, tuple): for f in funcs: - self.manager.addEvent(event, getattr(self,f)) + self.evm.addEvent(event, getattr(self,f)) else: - self.manager.addEvent(event, getattr(self,funcs)) + self.evm.addEvent(event, getattr(self,funcs)) #delete for various reasons self.event_map = None if self.event_list: for f in self.event_list: - self.manager.addEvent(f, getattr(self,f)) + self.evm.addEvent(f, getattr(self,f)) self.event_list = None self.initPeriodical() + self.init() self.setup() def initPeriodical(self): @@ -108,36 +121,51 @@ class Hook(Base): def __repr__(self): return "<Hook %s>" % self.__name__ - + + def isActivated(self): + """ checks if hook is activated""" + return self.getConfig("activated") + + def init(self): + pass + def setup(self): """ more init stuff if needed """ pass - def unload(self): - """ called when hook was deactivated """ + def activate(self): + """ Used to activate the hook """ + if has_method(self.__class__, "coreReady"): + self.logDebug("Deprecated method .coreReady() use activated() instead") + self.coreReady() + + def deactivate(self): + """ Used to deactivate the hook. """ pass - - def isActivated(self): - """ checks if hook is activated""" - return self.config.getPlugin(self.__name__, "activated") - - #event methods - overwrite these if needed - def coreReady(self): + def periodical(self): pass - def coreExiting(self): + def newCaptchaTask(self, task): + """ new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """ pass - + + def captchaCorrect(self, task): + pass + + def captchaInvalid(self, task): + pass + + # public events starts from here def downloadPreparing(self, pyfile): pass def downloadFinished(self, pyfile): pass - + def downloadFailed(self, pyfile): pass - + def packageFinished(self, pypack): pass @@ -145,17 +173,4 @@ class Hook(Base): pass def afterReconnecting(self, ip): - pass - - def periodical(self): - pass - - def newCaptchaTask(self, task): - """ new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """ - pass - - def captchaCorrect(self, task): - pass - - def captchaInvalid(self, task): pass
\ No newline at end of file diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py index 814a70949..c30fed412 100644 --- a/module/plugins/Hoster.py +++ b/module/plugins/Hoster.py @@ -13,21 +13,473 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay + + @author: RaNaN, spoob, mkaay """ -from module.plugins.Plugin import Plugin +from time import time, sleep +from random import randint + +import os + +if os.name != "nt": + from module.utils.fs import chown + from pwd import getpwnam + from grp import getgrnam + +from Base import Base, Fail, Retry +from module.utils import chunks as _chunks +from module.utils.fs import save_join, save_filename, fs_encode, fs_decode,\ + remove, makedirs, chmod, stat, exists, join + +# Import for Hoster Plugins +chunks = _chunks + +class Abort(Exception): + """ raised when aborted """ + +class Reconnect(Exception): + """ raised when reconnected """ + +class SkipDownload(Exception): + """ raised when download should be skipped """ + +class Hoster(Base): + """ + Base plugin for hoster plugin. Overwrite getInfo for online status retrieval, process for downloading. + """ + + @staticmethod + def getInfo(urls): + """This method is used to retrieve the online status of files for hoster plugins. + It has to *yield* list of tuples with the result in this format (name, size, status, url), + where status is one of API pyfile statusses. + + :param urls: List of urls + :return: yield list of tuple with results (name, size, status, url) + """ + pass + + def __init__(self, pyfile): + Base.__init__(self, pyfile.m.core) + + self.wantReconnect = False + #: enables simultaneous processing of multiple downloads + self.limitDL = 0 + #: chunk limit + self.chunkLimit = 1 + #: enables resume (will be ignored if server dont accept chunks) + self.resumeDownload = False + + #: time() + wait in seconds + self.waitUntil = 0 + self.waiting = False + + self.ocr = None #captcha reader instance + #: account handler instance, see :py:class:`Account` + self.account = self.core.accountManager.getAccountForPlugin(self.__name__) + + #: premium status + self.premium = False + #: username/login + self.user = None + + if self.account and not self.account.isUsable(): self.account = None + if self.account: + self.user = self.account.loginname + #: Browser instance, see `network.Browser` + self.req = self.account.getAccountRequest() + # Default: -1, True, True + self.chunkLimit, self.resumeDownload, self.multiDL = self.account.getDownloadSettings() + self.premium = self.account.isPremium() + else: + self.req = self.core.requestFactory.getRequest(self.__name__) + + #: associated pyfile instance, see `PyFile` + self.pyfile = pyfile + self.thread = None # holds thread in future + + #: location where the last call to download was saved + self.lastDownload = "" + #: re match of the last call to `checkDownload` + self.lastCheck = None + #: js engine, see `JsEngine` + self.js = self.core.js + self.cTask = None #captcha task + + self.retries = 0 # amount of retries already made + self.html = None # some plugins store html code here + + self.init() + + def getMultiDL(self): + self.logDebug("Deprectated attribute multiDL, use limitDL instead") + return self.limitDL <= 0 + + def setMultiDL(self, val): + self.logDebug("Deprectated attribute multiDL, use limitDL instead") + self.limitDL = 0 if val else 1 + + multiDL = property(getMultiDL, setMultiDL) + + def getChunkCount(self): + if self.chunkLimit <= 0: + return self.config["download"]["chunks"] + return min(self.config["download"]["chunks"], self.chunkLimit) + + def getDownloadLimit(self): + if self.account: + limit = self.account.options.get("limitDL", 0) + if limit == "": limit = 0 + return int(limit) + else: + return self.limitDL + + + def __call__(self): + return self.__name__ + + def init(self): + """initialize the plugin (in addition to `__init__`)""" + pass + + def setup(self): + """ setup for enviroment and other things, called before downloading (possibly more than one time)""" + pass + + def preprocessing(self, thread): + """ handles important things to do before starting """ + self.thread = thread + + if self.account: + # will force a relogin or reload of account info if necessary + self.account.getAccountInfo() + else: + self.req.clearCookies() + + self.setup() + + self.pyfile.setStatus("starting") + + return self.process(self.pyfile) + + + def process(self, pyfile): + """the 'main' method of every plugin, you **have to** overwrite it""" + raise NotImplementedError + + def resetAccount(self): + """ dont use account and retry download """ + self.account = None + self.req = self.core.requestFactory.getRequest(self.__name__) + self.retry() + + def checksum(self, local_file=None): + """ + return codes: + 0 - checksum ok + 1 - checksum wrong + 5 - can't get checksum + 10 - not implemented + 20 - unknown error + """ + #@TODO checksum check hook + + return True, 10 + + + def setWait(self, seconds, reconnect=False): + """Set a specific wait time later used with `wait` + + :param seconds: wait time in seconds + :param reconnect: True if a reconnect would avoid wait time + """ + if reconnect: + self.wantReconnect = True + self.pyfile.waitUntil = time() + int(seconds) + + def wait(self): + """ waits the time previously set """ + self.waiting = True + self.pyfile.setStatus("waiting") + + while self.pyfile.waitUntil > time(): + self.thread.m.reconnecting.wait(2) + + if self.pyfile.abort: raise Abort + if self.thread.m.reconnecting.isSet(): + self.waiting = False + self.wantReconnect = False + raise Reconnect + + self.waiting = False + self.pyfile.setStatus("starting") + + def offline(self): + """ fail and indicate file is offline """ + raise Fail("offline") + + def tempOffline(self): + """ fail and indicates file ist temporary offline, the core may take consequences """ + raise Fail("temp. offline") + + def retry(self, max_tries=3, wait_time=1, reason=""): + """Retries and begin again from the beginning + + :param max_tries: number of maximum retries + :param wait_time: time to wait in seconds + :param reason: reason for retrying, will be passed to fail if max_tries reached + """ + if 0 < max_tries <= self.retries: + if not reason: reason = "Max retries reached" + raise Fail(reason) + + self.wantReconnect = False + self.setWait(wait_time) + self.wait() + + self.retries += 1 + raise Retry(reason) + + def invalidCaptcha(self): + if self.cTask: + self.cTask.invalid() + + def correctCaptcha(self): + if self.cTask: + self.cTask.correct() + + def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', + result_type='textual'): + """ Loads a captcha and decrypts it with ocr, plugin, user input + + :param url: url of captcha image + :param get: get part for request + :param post: post part for request + :param cookies: True if cookies should be enabled + :param forceUser: if True, ocr is not used + :param imgtype: Type of the Image + :param result_type: 'textual' if text is written on the captcha\ + or 'positional' for captcha where the user have to click\ + on a specific region on the captcha + + :return: result of decrypting + """ + + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time())[-6:].replace(".", "") + temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + temp_file.write(img) + temp_file.close() + + has_plugin = self.__name__ in self.core.pluginManager.getPlugins("captcha") + + if self.core.captcha: + Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) + else: + Ocr = None + + if Ocr and not forceUser: + sleep(randint(3000, 5000) / 1000.0) + if self.pyfile.abort: raise Abort + + ocr = Ocr() + result = ocr.get_captcha(temp_file.name) + else: + captchaManager = self.core.captchaManager + task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) + self.cTask = task + captchaManager.handleCaptcha(task) + + while task.isWaiting(): + if self.pyfile.abort: + captchaManager.removeTask(task) + raise Abort + sleep(1) + + captchaManager.removeTask(task) + + if task.error and has_plugin: #ignore default error message since the user could use OCR + self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) + elif task.error: + self.fail(task.error) + elif not task.result: + self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) + + result = task.result + self.log.debug("Received captcha result: %s" % str(result)) + + if not self.core.debug: + try: + remove(temp_file.name) + except: + pass + + return result + + + def load(self, *args, **kwargs): + """ See 'Base' load method for more info """ + if self.pyfile.abort: raise Abort + return Base.load(self, *args, **kwargs) + + def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): + """Downloads the content at url to download folder + + :param url: + :param get: + :param post: + :param ref: + :param cookies: + :param disposition: if True and server provides content-disposition header\ + the filename will be changed if needed + :return: The location where the file was saved + """ + + self.checkForSameFiles() + + self.pyfile.setStatus("downloading") + + download_folder = self.config['general']['download_folder'] + + location = save_join(download_folder, self.pyfile.package().folder) + + if not exists(location): + makedirs(location, int(self.core.config["permission"]["folder"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(location, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + # convert back to unicode + location = fs_decode(location) + name = save_filename(self.pyfile.name) + + filename = join(location, name) + + self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) + + try: + newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, + chunks=self.getChunkCount(), resume=self.resumeDownload, + progressNotify=self.pyfile.setProgress, disposition=disposition) + finally: + self.pyfile.size = self.req.size + + if disposition and newname and newname != name: #triple check, just to be sure + self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) + self.pyfile.name = newname + filename = join(location, newname) + + fs_filename = fs_encode(filename) + + if self.core.config["permission"]["change_file"]: + chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) + + if self.core.config["permission"]["change_dl"] and os.name != "nt": + try: + uid = getpwnam(self.config["permission"]["user"])[2] + gid = getgrnam(self.config["permission"]["group"])[2] + + chown(fs_filename, uid, gid) + except Exception, e: + self.log.warning(_("Setting User and Group failed: %s") % str(e)) + + self.lastDownload = filename + return self.lastDownload + + def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): + """ checks the content of the last downloaded file, re match is saved to `lastCheck` + + :param rules: dict with names and rules to match (compiled regexp or strings) + :param api_size: expected file size + :param max_size: if the file is larger then it wont be checked + :param delete: delete if matched + :param read_size: amount of bytes to read from files larger then max_size + :return: dictionary key of the first rule that matched + """ + lastDownload = fs_encode(self.lastDownload) + if not exists(lastDownload): return None + + size = stat(lastDownload) + size = size.st_size + + if api_size and api_size <= size: return None + elif size > max_size and not read_size: return None + self.log.debug("Download Check triggered") + f = open(lastDownload, "rb") + content = f.read(read_size if read_size else -1) + f.close() + #produces encoding errors, better log to other file in the future? + #self.log.debug("Content: %s" % content) + for name, rule in rules.iteritems(): + if type(rule) in (str, unicode): + if rule in content: + if delete: + remove(lastDownload) + return name + elif hasattr(rule, "search"): + m = rule.search(content) + if m: + if delete: + remove(lastDownload) + self.lastCheck = m + return name + + + def getPassword(self): + """ get the password the user provided in the package""" + password = self.pyfile.package().password + if not password: return "" + return password + + + def checkForSameFiles(self, starting=False): + """ checks if same file was/is downloaded within same package + + :param starting: indicates that the current download is going to start + :raises SkipDownload: + """ + + pack = self.pyfile.package() + + for pyfile in self.core.files.cache.values(): + if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: + if pyfile.status in (0, 12): #finished or downloading + raise SkipDownload(pyfile.pluginname) + elif pyfile.status in ( + 5, 7) and starting: #a download is waiting/starting and was appenrently started before + raise SkipDownload(pyfile.pluginname) + + download_folder = self.config['general']['download_folder'] + location = save_join(download_folder, pack.folder, self.pyfile.name) + + if starting and self.core.config['download']['skip_existing'] and exists(location): + size = os.stat(location).st_size + if size >= self.pyfile.size: + raise SkipDownload("File exists.") + + pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) + if pyfile: + if exists(location): + raise SkipDownload(pyfile[0]) -def getInfo(self): - #result = [ .. (name, size, status, url) .. ] - return + self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) -class Hoster(Plugin): - __name__ = "Hoster" - __version__ = "0.1" - __pattern__ = None - __type__ = "hoster" - __description__ = """Base hoster plugin""" - __author_name__ = ("mkaay") - __author_mail__ = ("mkaay@mkaay.de") + def clean(self): + """ clean everything and remove references """ + if hasattr(self, "pyfile"): + del self.pyfile + if hasattr(self, "req"): + self.req.close() + del self.req + if hasattr(self, "thread"): + del self.thread + if hasattr(self, "html"): + del self.html diff --git a/module/plugins/MultiHoster.py b/module/plugins/MultiHoster.py new file mode 100644 index 000000000..abbc14466 --- /dev/null +++ b/module/plugins/MultiHoster.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +from time import time + +from module.utils import remove_chars + +from Account import Account + +def normalize(domain): + """ Normalize domain/plugin name, so they are comparable """ + return remove_chars(domain.strip().lower(), "-.") + +#noinspection PyUnresolvedReferences +class MultiHoster(Account): + """ + Base class for MultiHoster services. + This is also an Account instance so you should see :class:`Account` and overwrite necessary methods. + Multihoster becomes only active when an Account was entered and the MultiHoster hook was activated. + You need to overwrite `loadHosterList` and a corresponding :class:`Hoster` plugin with the same name should + be available to make your service working. + """ + + #: List of hoster names that will be replaced so pyLoad will recognize them: (orig_name, pyload_name) + replacements = [("freakshare.net", "freakshare.com")] + + #: Load new hoster list every x seconds + hoster_timeout = 300 + + def __init__(self, *args, **kwargs): + + # Hoster list + self.hoster = [] + # Timestamp + self.ts = 0 + + Account.__init__(self, *args, **kwargs) + + def loadHosterList(self, req): + """Load list of supported hoster + + :return: List of domain names + """ + raise NotImplementedError + + + def isHosterUsuable(self, domain): + """ Determine before downloading if hoster should be used. + + :param domain: domain name + :return: True to let the MultiHoster download, False to fallback to default plugin + """ + return True + + def getHosterList(self, force=False): + if self.ts + self.hoster_timeout < time() or force: + req = self.getAccountRequest() + try: + self.hoster = self.loadHosterList(req) + except Exception, e: + self.logError(e) + return [] + finally: + req.close() + + for rep in self.replacements: + if rep[0] in self.hoster: + self.hoster.remove(rep[0]) + if rep[1] not in self.hoster: + self.hoster.append(rep[1]) + + self.ts = time() + + return self.hoster
\ No newline at end of file diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py deleted file mode 100644 index 15bf3971f..000000000 --- a/module/plugins/Plugin.py +++ /dev/null @@ -1,617 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN, spoob, mkaay -""" - -from time import time, sleep -from random import randint - -import os -from os import remove, makedirs, chmod, stat -from os.path import exists, join - -if os.name != "nt": - from os import chown - from pwd import getpwnam - from grp import getgrnam - -from itertools import islice - -from module.utils import save_join, save_path, fs_encode, fs_decode - -def chunks(iterable, size): - it = iter(iterable) - item = list(islice(it, size)) - while item: - yield item - item = list(islice(it, size)) - - -class Abort(Exception): - """ raised when aborted """ - - -class Fail(Exception): - """ raised when failed """ - - -class Reconnect(Exception): - """ raised when reconnected """ - - -class Retry(Exception): - """ raised when start again from beginning """ - - -class SkipDownload(Exception): - """ raised when download should be skipped """ - - -class Base(object): - """ - A Base class with log/config/db methods *all* plugin types can use - """ - - def __init__(self, core): - #: Core instance - self.core = core - #: logging instance - self.log = core.log - #: core config - self.config = core.config - - #log functions - def logInfo(self, *args): - self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logWarning(self, *args): - self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logError(self, *args): - self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - def logDebug(self, *args): - self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args]))) - - - def setConf(self, option, value): - """ see `setConfig` """ - self.core.config.setPlugin(self.__name__, option, value) - - def setConfig(self, option, value): - """ Set config value for current plugin - - :param option: - :param value: - :return: - """ - self.setConf(option, value) - - def getConf(self, option): - """ see `getConfig` """ - return self.core.config.getPlugin(self.__name__, option) - - def getConfig(self, option): - """ Returns config value for current plugin - - :param option: - :return: - """ - return self.getConf(option) - - def setStorage(self, key, value): - """ Saves a value persistently to the database """ - self.core.db.setStorage(self.__name__, key, value) - - def store(self, key, value): - """ same as `setStorage` """ - self.core.db.setStorage(self.__name__, key, value) - - def getStorage(self, key=None, default=None): - """ Retrieves saved value or dict of all saved entries if key is None """ - if key is not None: - return self.core.db.getStorage(self.__name__, key) or default - return self.core.db.getStorage(self.__name__, key) - - def retrieve(self, *args, **kwargs): - """ same as `getStorage` """ - return self.getStorage(*args, **kwargs) - - def delStorage(self, key): - """ Delete entry in db """ - self.core.db.delStorage(self.__name__, key) - - -class Plugin(Base): - """ - Base plugin for hoster/crypter. - Overwrite `process` / `decrypt` in your subclassed plugin. - """ - __name__ = "Plugin" - __version__ = "0.4" - __pattern__ = None - __type__ = "hoster" - __config__ = [("name", "type", "desc", "default")] - __description__ = """Base Plugin""" - __author_name__ = ("RaNaN", "spoob", "mkaay") - __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de") - - def __init__(self, pyfile): - Base.__init__(self, pyfile.m.core) - - self.wantReconnect = False - #: enables simultaneous processing of multiple downloads - self.multiDL = True - self.limitDL = 0 - #: chunk limit - self.chunkLimit = 1 - self.resumeDownload = False - - #: time() + wait in seconds - self.waitUntil = 0 - self.waiting = False - - self.ocr = None #captcha reader instance - #: account handler instance, see :py:class:`Account` - self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) - - #: premium status - self.premium = False - #: username/login - self.user = None - - if self.account and not self.account.canUse(): self.account = None - if self.account: - self.user, data = self.account.selectAccount() - #: Browser instance, see `network.Browser` - self.req = self.account.getAccountRequest(self.user) - self.chunkLimit = -1 # chunk limit, -1 for unlimited - #: enables resume (will be ignored if server dont accept chunks) - self.resumeDownload = True - self.multiDL = True #every hoster with account should provide multiple downloads - #: premium status - self.premium = self.account.isPremium(self.user) - else: - self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) - - #: associated pyfile instance, see `PyFile` - self.pyfile = pyfile - self.thread = None # holds thread in future - - #: location where the last call to download was saved - self.lastDownload = "" - #: re match of the last call to `checkDownload` - self.lastCheck = None - #: js engine, see `JsEngine` - self.js = self.core.js - self.cTask = None #captcha task - - self.retries = 0 # amount of retries already made - self.html = None # some plugins store html code here - - self.init() - - def getChunkCount(self): - if self.chunkLimit <= 0: - return self.config["download"]["chunks"] - return min(self.config["download"]["chunks"], self.chunkLimit) - - def __call__(self): - return self.__name__ - - def init(self): - """initialize the plugin (in addition to `__init__`)""" - pass - - def setup(self): - """ setup for enviroment and other things, called before downloading (possibly more than one time)""" - pass - - def preprocessing(self, thread): - """ handles important things to do before starting """ - self.thread = thread - - if self.account: - self.account.checkLogin(self.user) - else: - self.req.clearCookies() - - self.setup() - - self.pyfile.setStatus("starting") - - return self.process(self.pyfile) - - - def process(self, pyfile): - """the 'main' method of every plugin, you **have to** overwrite it""" - raise NotImplementedError - - def resetAccount(self): - """ dont use account and retry download """ - self.account = None - self.req = self.core.requestFactory.getRequest(self.__name__) - self.retry() - - def checksum(self, local_file=None): - """ - return codes: - 0 - checksum ok - 1 - checksum wrong - 5 - can't get checksum - 10 - not implemented - 20 - unknown error - """ - #@TODO checksum check hook - - return True, 10 - - - def setWait(self, seconds, reconnect=False): - """Set a specific wait time later used with `wait` - - :param seconds: wait time in seconds - :param reconnect: True if a reconnect would avoid wait time - """ - if reconnect: - self.wantReconnect = True - self.pyfile.waitUntil = time() + int(seconds) - - def wait(self): - """ waits the time previously set """ - self.waiting = True - self.pyfile.setStatus("waiting") - - while self.pyfile.waitUntil > time(): - self.thread.m.reconnecting.wait(2) - - if self.pyfile.abort: raise Abort - if self.thread.m.reconnecting.isSet(): - self.waiting = False - self.wantReconnect = False - raise Reconnect - - self.waiting = False - self.pyfile.setStatus("starting") - - def fail(self, reason): - """ fail and give reason """ - raise Fail(reason) - - def offline(self): - """ fail and indicate file is offline """ - raise Fail("offline") - - def tempOffline(self): - """ fail and indicates file ist temporary offline, the core may take consequences """ - raise Fail("temp. offline") - - def retry(self, max_tries=3, wait_time=1, reason=""): - """Retries and begin again from the beginning - - :param max_tries: number of maximum retries - :param wait_time: time to wait in seconds - :param reason: reason for retrying, will be passed to fail if max_tries reached - """ - if 0 < max_tries <= self.retries: - if not reason: reason = "Max retries reached" - raise Fail(reason) - - self.wantReconnect = False - self.setWait(wait_time) - self.wait() - - self.retries += 1 - raise Retry(reason) - - def invalidCaptcha(self): - if self.cTask: - self.cTask.invalid() - - def correctCaptcha(self): - if self.cTask: - self.cTask.correct() - - def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', - result_type='textual'): - """ Loads a captcha and decrypts it with ocr, plugin, user input - - :param url: url of captcha image - :param get: get part for request - :param post: post part for request - :param cookies: True if cookies should be enabled - :param forceUser: if True, ocr is not used - :param imgtype: Type of the Image - :param result_type: 'textual' if text is written on the captcha\ - or 'positional' for captcha where the user have to click\ - on a specific region on the captcha - - :return: result of decrypting - """ - - img = self.load(url, get=get, post=post, cookies=cookies) - - id = ("%.2f" % time())[-6:].replace(".", "") - temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") - temp_file.write(img) - temp_file.close() - - has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins - - if self.core.captcha: - Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) - else: - Ocr = None - - if Ocr and not forceUser: - sleep(randint(3000, 5000) / 1000.0) - if self.pyfile.abort: raise Abort - - ocr = Ocr() - result = ocr.get_captcha(temp_file.name) - else: - captchaManager = self.core.captchaManager - task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) - self.cTask = task - captchaManager.handleCaptcha(task) - - while task.isWaiting(): - if self.pyfile.abort: - captchaManager.removeTask(task) - raise Abort - sleep(1) - - captchaManager.removeTask(task) - - if task.error and has_plugin: #ignore default error message since the user could use OCR - self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) - elif task.error: - self.fail(task.error) - elif not task.result: - self.fail(_("No captcha result obtained in appropiate time by any of the plugins.")) - - result = task.result - self.log.debug("Received captcha result: %s" % str(result)) - - if not self.core.debug: - try: - remove(temp_file.name) - except: - pass - - return result - - - def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): - """Load content at url and returns it - - :param url: - :param get: - :param post: - :param ref: - :param cookies: - :param just_header: if True only the header will be retrieved and returned as dict - :param decode: Wether to decode the output according to http header, should be True in most cases - :return: Loaded content - """ - if self.pyfile.abort: raise Abort - #utf8 vs decode -> please use decode attribute in all future plugins - if type(url) == unicode: url = str(url) - - res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) - - if self.core.debug: - from inspect import currentframe - - frame = currentframe() - if not exists(join("tmp", self.__name__)): - makedirs(join("tmp", self.__name__)) - - f = open( - join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) - , "wb") - del frame # delete the frame or it wont be cleaned - - try: - tmp = res.encode("utf8") - except: - tmp = res - - f.write(tmp) - f.close() - - if just_header: - #parse header - header = {"code": self.req.code} - for line in res.splitlines(): - line = line.strip() - if not line or ":" not in line: continue - - key, none, value = line.partition(":") - key = key.lower().strip() - value = value.strip() - - if key in header: - if type(header[key]) == list: - header[key].append(value) - else: - header[key] = [header[key], value] - else: - header[key] = value - res = header - - return res - - def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): - """Downloads the content at url to download folder - - :param url: - :param get: - :param post: - :param ref: - :param cookies: - :param disposition: if True and server provides content-disposition header\ - the filename will be changed if needed - :return: The location where the file was saved - """ - - self.checkForSameFiles() - - self.pyfile.setStatus("downloading") - - download_folder = self.config['general']['download_folder'] - - location = save_join(download_folder, self.pyfile.package().folder) - - if not exists(location): - makedirs(location, int(self.core.config["permission"]["folder"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - - chown(location, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) - - # convert back to unicode - location = fs_decode(location) - name = save_path(self.pyfile.name) - - filename = join(location, name) - - self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) - - try: - newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, - chunks=self.getChunkCount(), resume=self.resumeDownload, - progressNotify=self.pyfile.setProgress, disposition=disposition) - finally: - self.pyfile.size = self.req.size - - if disposition and newname and newname != name: #triple check, just to be sure - self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname}) - self.pyfile.name = newname - filename = join(location, newname) - - fs_filename = fs_encode(filename) - - if self.core.config["permission"]["change_file"]: - chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) - - if self.core.config["permission"]["change_dl"] and os.name != "nt": - try: - uid = getpwnam(self.config["permission"]["user"])[2] - gid = getgrnam(self.config["permission"]["group"])[2] - - chown(fs_filename, uid, gid) - except Exception, e: - self.log.warning(_("Setting User and Group failed: %s") % str(e)) - - self.lastDownload = filename - return self.lastDownload - - def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): - """ checks the content of the last downloaded file, re match is saved to `lastCheck` - - :param rules: dict with names and rules to match (compiled regexp or strings) - :param api_size: expected file size - :param max_size: if the file is larger then it wont be checked - :param delete: delete if matched - :param read_size: amount of bytes to read from files larger then max_size - :return: dictionary key of the first rule that matched - """ - lastDownload = fs_encode(self.lastDownload) - if not exists(lastDownload): return None - - size = stat(lastDownload) - size = size.st_size - - if api_size and api_size <= size: return None - elif size > max_size and not read_size: return None - self.log.debug("Download Check triggered") - f = open(lastDownload, "rb") - content = f.read(read_size if read_size else -1) - f.close() - #produces encoding errors, better log to other file in the future? - #self.log.debug("Content: %s" % content) - for name, rule in rules.iteritems(): - if type(rule) in (str, unicode): - if rule in content: - if delete: - remove(lastDownload) - return name - elif hasattr(rule, "search"): - m = rule.search(content) - if m: - if delete: - remove(lastDownload) - self.lastCheck = m - return name - - - def getPassword(self): - """ get the password the user provided in the package""" - password = self.pyfile.package().password - if not password: return "" - return password - - - def checkForSameFiles(self, starting=False): - """ checks if same file was/is downloaded within same package - - :param starting: indicates that the current download is going to start - :raises SkipDownload: - """ - - pack = self.pyfile.package() - - for pyfile in self.core.files.cache.values(): - if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: - if pyfile.status in (0, 12): #finished or downloading - raise SkipDownload(pyfile.pluginname) - elif pyfile.status in ( - 5, 7) and starting: #a download is waiting/starting and was appenrently started before - raise SkipDownload(pyfile.pluginname) - - download_folder = self.config['general']['download_folder'] - location = save_join(download_folder, pack.folder, self.pyfile.name) - - if starting and self.core.config['download']['skip_existing'] and exists(location): - size = os.stat(location).st_size - if size >= self.pyfile.size: - raise SkipDownload("File exists.") - - pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) - if pyfile: - if exists(location): - raise SkipDownload(pyfile[0]) - - self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name) - - def clean(self): - """ clean everything and remove references """ - if hasattr(self, "pyfile"): - del self.pyfile - if hasattr(self, "req"): - self.req.close() - del self.req - if hasattr(self, "thread"): - del self.thread - if hasattr(self, "html"): - del self.html diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py index f3f5f47bc..c345f765f 100644 --- a/module/plugins/PluginManager.py +++ b/module/plugins/PluginManager.py @@ -21,24 +21,34 @@ import re import sys from os import listdir, makedirs -from os.path import isfile, join, exists, abspath +from os.path import isfile, join, exists, abspath, basename from sys import version_info -from itertools import chain +from time import time from traceback import print_exc from module.lib.SafeEval import const_eval as literal_eval -from module.ConfigParser import IGNORE +from module.plugins.Base import Base + +from new_collections import namedtuple + +# ignore these plugin configs, mainly because plugins were wiped out +IGNORE = ( + "FreakshareNet", "SpeedManager", "ArchiveTo", "ShareCx", ('hooks', 'UnRar'), + 'EasyShareCom', 'FlyshareCz' + ) + +PluginTuple = namedtuple("PluginTuple", "version re deps user path") class PluginManager: ROOT = "module.plugins." USERROOT = "userplugins." - TYPES = ("crypter", "container", "hoster", "captcha", "accounts", "hooks", "internal") + TYPES = ("crypter", "hoster", "captcha", "accounts", "hooks", "internal") - PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') - VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)') - CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) - DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)') + SINGLE = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*(?:r|u|_)?((?:(?<!")"(?!")|\'|\().*(?:(?<!")"(?!")|\'|\)))', + re.I) + # note the nongreedy character, that means we can not embed list and dicts + MULTI = re.compile(r'__(?P<attr>[a-z0-9_]+)__\s*=\s*((?:\{|\[|"{3}).*?(?:"""|\}|\]))', re.DOTALL | re.M | re.I) def __init__(self, core): self.core = core @@ -47,15 +57,23 @@ class PluginManager: self.log = core.log self.plugins = {} + self.modules = {} # cached modules + self.history = [] # match history to speedup parsing (type, name) self.createIndex() + + self.core.config.parseValues(self.core.config.PLUGIN) + #register for import hook sys.meta_path.append(self) + def logDebug(self, type, plugin, msg): + self.log.debug("Plugin %s | %s: %s" % (type, plugin, msg)) + def createIndex(self): """create information for all plugins available""" - + # add to path, so we can import from userplugins sys.path.append(abspath("")) if not exists("userplugins"): @@ -64,27 +82,14 @@ class PluginManager: f = open(join("userplugins", "__init__.py"), "wb") f.close() - self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) - - self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") - self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") - self.plugins["hooks"] = self.hookPlugins = self.parse("hooks") - self.plugins["internal"] = self.internalPlugins = self.parse("internal") + a = time() + for type in self.TYPES: + self.plugins[type] = self.parse(type) - self.log.debug("created index of plugins") + self.log.debug("Created index of plugins in %.2f ms", (time() - a) * 1000) - def parse(self, folder, pattern=False, home={}): - """ - returns dict with information - home contains parsed plugins from module. - - { - name : {path, version, config, (pattern, re), (plugin, class)} - } - - """ + def parse(self, folder, home=None): + """ Analyze and parses all plugins in folder """ plugins = {} if home: pfolder = join("userplugins", folder) @@ -100,10 +105,6 @@ class PluginManager: for f in listdir(pfolder): if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith( "_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - data = open(join(pfolder, f)) - content = data.read() - data.close() - if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): continue elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): @@ -111,148 +112,168 @@ class PluginManager: elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): continue + # replace suffix and version tag name = f[:-3] if name[-1] == ".": name = name[:-4] - version = self.VERSION.findall(content) - if version: - version = float(version[0][1]) - else: - version = 0 + plugin = self.parsePlugin(join(pfolder, f), folder, name, home) + if plugin: + plugins[name] = plugin - # home contains plugins from pyload root - if home and name in home: - if home[name]["v"] >= version: - continue + if not home: + temp = self.parse(folder, plugins) + plugins.update(temp) - if name in IGNORE or (folder, name) in IGNORE: - continue + return plugins - plugins[name] = {} - plugins[name]["v"] = version + def parsePlugin(self, filename, folder, name, home=None): + """ Parses a plugin from disk, folder means plugin type in this context. Also sets config. - module = f.replace(".pyc", "").replace(".py", "") + :arg home: dict with plugins, of which the found one will be matched against (according version) + :returns PluginTuple""" - # the plugin is loaded from user directory - plugins[name]["user"] = True if home else False - plugins[name]["name"] = module + data = open(filename, "rb") + content = data.read() + data.close() - if pattern: - pattern = self.PATTERN.findall(content) + attrs = {} + for m in self.SINGLE.findall(content) + self.MULTI.findall(content): + #replace gettext function and eval result + try: + attrs[m[0]] = literal_eval(m[-1].replace("_(", "(")) + except: + self.logDebug(folder, name, "Error when parsing: %s" % m[-1]) + return + if not hasattr(Base, "__%s__" % m[0]): + if m[0] != "type": #TODO remove type from all plugins, its not needed + self.logDebug(folder, name, "Unknown attribute '%s'" % m[0]) - if pattern: - pattern = pattern[0][1] - else: - pattern = "^unmachtable$" + version = 0 - plugins[name]["pattern"] = pattern + if "version" in attrs: + try: + version = float(attrs["version"]) + except ValueError: + self.logDebug(folder, name, "Invalid version %s" % attrs["version"]) + version = 9 #TODO remove when plugins are fixed, causing update loops + else: + self.logDebug(folder, name, "No version attribute") - try: - plugins[name]["re"] = re.compile(pattern) - except: - self.log.error(_("%s has a invalid pattern.") % name) + # home contains plugins from pyload root + if home and name in home: + if home[name].version >= version: + return + if name in IGNORE or (folder, name) in IGNORE: + return - # internals have no config - if folder == "internal": - self.core.config.deleteConfig(name) - continue + if "pattern" in attrs and attrs["pattern"]: + try: + plugin_re = re.compile(attrs["pattern"]) + except: + self.logDebug(folder, name, "Invalid regexp pattern '%s'" % attrs["pattern"]) + plugin_re = None + else: plugin_re = None - config = self.CONFIG.findall(content) - if config: - config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" + deps = attrs.get("dependencies", None) - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] + # create plugin tuple + plugin = PluginTuple(version, plugin_re, deps, bool(home), filename) - if folder == "hooks": - append = True - for item in config: - if item[0] == "activated": append = False - # activated flag missing - if append: config.append(["activated", "bool", "Activated", False]) + # internals have no config + if folder == "internal": + return plugin - try: - self.core.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) + if folder == "hooks" and "config" not in attrs: + attrs["config"] = (["activated", "bool", "Activated", False],) - elif folder == "hooks": #force config creation - desc = self.DESC.findall(content) - desc = desc[0][1] if desc else "" - config = (["activated", "bool", "Activated", False],) + if "config" in attrs and attrs["config"]: + config = attrs["config"] + desc = attrs.get("description", "") + long_desc = attrs.get("long_description", "") - try: - self.core.config.addPluginConfig(name, config, desc) - except: - self.log.error("Invalid config in %s: %s" % (name, config)) + if type(config[0]) == tuple: + config = [list(x) for x in config] + else: + config = [list(config)] - if not home: - temp = self.parse(folder, pattern, plugins) - plugins.update(temp) + if folder == "hooks": + append = True + for item in config: + if item[0] == "activated": append = False - return plugins + # activated flag missing + if append: config.insert(0, ("activated", "bool", "Activated", False)) + + try: + self.core.config.addConfigSection(name, name, desc, long_desc, config) + except: + self.logDebug(folder, name, "Invalid config %s" % config) + + return plugin def parseUrls(self, urls): - """parse plugins for given list of urls""" + """parse plugins for given list of urls, seperate to crypter and hoster""" - last = None - res = [] # tupels of (url, plugin) + res = {"hoster": [], "crypter": []} # tupels of (url, plugin) for url in urls: - if type(url) not in (str, unicode, buffer): continue + if type(url) not in (str, unicode, buffer): + self.log.debug("Parsing invalid type %s" % type(url)) + continue + found = False - if last and last[1]["re"].match(url): - res.append((url, last[0])) + for ptype, name in self.history: + if self.plugins[ptype][name].re.match(url): + res[ptype].append((url, name)) + found = (ptype, name) + break + + if found: # found match + if self.history[0] != found: #update history + self.history.remove(found) + self.history.insert(0, found) continue - for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(), - self.containerPlugins.iteritems()): - if value["re"].match(url): - res.append((url, name)) - last = (name, value) - found = True - break + for ptype in ("crypter", "hoster"): + for name, plugin in self.plugins[ptype].iteritems(): + if plugin.re.match(url): + res[ptype].append((url, name)) + self.history.insert(0, (ptype, name)) + del self.history[10:] # cut down to size of 10 + found = True + break if not found: - res.append((url, "BasePlugin")) + res["hoster"].append((url, "BasePlugin")) + + return res["hoster"], res["crypter"] - return res + def getPlugins(self, type): + return self.plugins.get(type, None) - def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")): + def findPlugin(self, name, pluginlist=("hoster", "crypter")): for ptype in pluginlist: if name in self.plugins[ptype]: - return self.plugins[ptype][name], ptype + return ptype, self.plugins[ptype][name] return None, None - def getPlugin(self, name, original=False): - """return plugin module from hoster|decrypter|container""" - plugin, type = self.findPlugin(name) - - if not plugin: - self.log.warning("Plugin %s not found." % name) - plugin = self.hosterPlugins["BasePlugin"] - - if "new_module" in plugin and not original: - return plugin["new_module"] - + def getPluginModule(self, name): + """ Decprecated: return plugin module from hoster|crypter""" + self.log.debug("Deprecated method: .getPluginModule()") + type, plugin = self.findPlugin(name) return self.loadModule(type, name) - def getPluginName(self, name): - """ used to obtain new name if other plugin was injected""" - plugin, type = self.findPlugin(name) + def getPluginClass(self, name): + """ return plugin class from hoster|crypter, always the not overwritten one """ + type, plugin = self.findPlugin(name) + return self.loadClass(type, name) - if "new_name" in plugin: - return plugin["new_name"] - - return name + # MultiHoster will overwrite this + getPlugin = getPluginClass def loadModule(self, type, name): """ Returns loaded module for plugin @@ -262,11 +283,12 @@ class PluginManager: """ plugins = self.plugins[type] if name in plugins: - if "module" in plugins[name]: return plugins[name]["module"] + if (type, name) in self.modules: return self.modules[(type, name)] try: - module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]["name"]), globals(), locals(), - plugins[name]["name"]) - plugins[name]["module"] = module #cache import, maybe unneeded + # convert path to python recognizable import + path = basename(plugins[name].path).replace(".pyc", "").replace(".py", "") + module = __import__(self.ROOT + "%s.%s" % (type, path), globals(), locals(), path) + self.modules[(type, name)] = module # cache import, maybe unneeded return module except Exception, e: self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)}) @@ -278,10 +300,6 @@ class PluginManager: module = self.loadModule(type, name) if module: return getattr(module, name) - def getAccountPlugins(self): - """return list of account plugin names""" - return self.accountPlugins.keys() - def find_module(self, fullname, path=None): #redirecting imports if necesarry if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins @@ -294,10 +312,10 @@ class PluginManager: if type in self.plugins and name in self.plugins[type]: #userplugin is a newer version - if not user and self.plugins[type][name]["user"]: + if not user and self.plugins[type][name].user: return self - #imported from userdir, but pyloads is newer - if user and not self.plugins[type][name]["user"]: + #imported from userdir, but pyloads is newer + if user and not self.plugins[type][name].user: return self @@ -329,7 +347,7 @@ class PluginManager: self.log.debug("Request reload of plugins: %s" % type_plugins) as_dict = {} - for t,n in type_plugins: + for t, n in type_plugins: if t in as_dict: as_dict[t].append(n) else: @@ -342,16 +360,13 @@ class PluginManager: for type in as_dict.iterkeys(): for plugin in as_dict[type]: if plugin in self.plugins[type]: - if "module" in self.plugins[type][plugin]: + if (type, plugin) in self.modules: self.log.debug("Reloading %s" % plugin) - reload(self.plugins[type][plugin]["module"]) + reload(self.modules[(type, plugin)]) - #index creation - self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True) - self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True) - self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True) - self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha") - self.plugins["accounts"] = self.accountPlugins = self.parse("accounts") + # index re-creation + for type in ("crypter", "container", "hoster", "captcha", "accounts"): + self.plugins[type] = self.parse(type) if "accounts" in as_dict: #accounts needs to be reloaded self.core.accountManager.initPlugins() @@ -359,22 +374,21 @@ class PluginManager: return True + def loadIcons(self): + """Loads all icons from plugins, plugin type is not in result, because its not important here. + :return: Dict of names mapped to icons + """ + pass -if __name__ == "__main__": - _ = lambda x: x - pypath = "/home/christian/Projekte/pyload-0.4/module/plugins" - - from time import time - - p = PluginManager(None) - - a = time() - - test = ["http://www.youtube.com/watch?v=%s" % x for x in range(0, 100)] - print p.parseUrls(test) + def loadIcon(self, type, name): + """ load icon for single plugin, base64 encoded""" + pass - b = time() + def checkDependencies(self, type, name): + """ Check deps for given plugin - print b - a, "s" + :return: List of unfullfilled dependencies + """ + pass diff --git a/module/plugins/accounts/FilesonicCom.py b/module/plugins/accounts/FilesonicCom.py index 1207f1b56..1b0104b2a 100644 --- a/module/plugins/accounts/FilesonicCom.py +++ b/module/plugins/accounts/FilesonicCom.py @@ -37,10 +37,10 @@ class FilesonicCom(Account): decode=True) return json_loads(xml)["FSApi_Utility"]["getFilesonicDomainForCurrentIp"]["response"] - def loadAccountInfo(self, user, req): + def loadAccountInfo(self, req): xml = req.load(self.API_URL + "/user?method=getInfo&format=json", - post={"u": user, - "p": self.accounts[user]["password"]}, decode=True) + post={"u": self.loginname, + "p": self.password}, decode=True) self.logDebug("account status retrieved from api %s" % xml) @@ -56,15 +56,16 @@ class FilesonicCom(Account): validuntil = -1 return {"validuntil": validuntil, "trafficleft": -1, "premium": premium} - def login(self, user, data, req): + def login(self, req): domain = self.getDomain(req) post_vars = { - "email": user, - "password": data["password"], + "email": self.loginname, + "password": self.password, "rememberMe": 1 } page = req.load("http://www%s/user/login" % domain, cookies=True, post=post_vars, decode=True) if "Provided password does not match." in page or "You must be logged in to view this page." in page: self.wrongPassword() + diff --git a/module/plugins/accounts/OronCom.py b/module/plugins/accounts/OronCom.py index 793984121..1fe8a4449 100755 --- a/module/plugins/accounts/OronCom.py +++ b/module/plugins/accounts/OronCom.py @@ -23,13 +23,13 @@ from time import strptime, mktime class OronCom(Account): __name__ = "OronCom" - __version__ = "0.12" + __version__ = "0.13" __type__ = "account" __description__ = """oron.com account plugin""" __author_name__ = ("DHMH") __author_mail__ = ("DHMH@pyload.org") - def loadAccountInfo(self, user, req): + def loadAccountInfo(self, req): req.load("http://oron.com/?op=change_lang&lang=german") src = req.load("http://oron.com/?op=my_account").replace("\n", "") validuntil = re.search(r"<td>Premiumaccount läuft bis:</td>\s*<td>(.*?)</td>", src) @@ -47,8 +47,9 @@ class OronCom(Account): tmp = {"validuntil": validuntil, "trafficleft": trafficleft, "premium" : premium} return tmp - def login(self, user, data, req): + def login(self, req): req.load("http://oron.com/?op=change_lang&lang=german") - page = req.load("http://oron.com/login", post={"login": user, "password": data["password"], "op": "login"}) + page = req.load("http://oron.com/login", post={"login": self.loginname, "password": self.password, "op": "login"}) if r'<b class="err">Login oder Passwort falsch</b>' in page: self.wrongPassword() + diff --git a/module/plugins/accounts/Premium4Me.py b/module/plugins/accounts/Premium4Me.py index de4fdc219..6a52cb61a 100644 --- a/module/plugins/accounts/Premium4Me.py +++ b/module/plugins/accounts/Premium4Me.py @@ -1,23 +1,27 @@ -from module.plugins.Account import Account
+# -*- coding: utf-8 -*-
+from module.plugins.MultiHoster import MultiHoster
-class Premium4Me(Account):
+class Premium4Me(MultiHoster):
__name__ = "Premium4Me"
- __version__ = "0.02"
+ __version__ = "0.10"
__type__ = "account"
__description__ = """Premium4.me account plugin"""
__author_name__ = ("RaNaN", "zoidberg")
__author_mail__ = ("RaNaN@pyload.org", "zoidberg@mujmail.cz")
- def loadAccountInfo(self, user, req):
+ def loadAccountInfo(self, req):
traffic = req.load("http://premium4.me/api/traffic.php?authcode=%s" % self.authcode)
- account_info = {"trafficleft": int(traffic) / 1024,
- "validuntil": -1}
+ account_info = {"trafficleft": int(traffic) / 1024, "validuntil": -1}
return account_info
- def login(self, user, data, req):
- self.authcode = req.load("http://premium4.me/api/getauthcode.php?username=%s&password=%s" % (user, data["password"])).strip()
-
+ def login(self, req):
+ self.authcode = req.load("http://premium4.me/api/getauthcode.php?username=%s&password=%s" % (self.loginname, self.password)).strip()
+
if "wrong username" in self.authcode:
- self.wrongPassword()
\ No newline at end of file + self.wrongPassword()
+
+ def loadHosterList(self, req):
+ page = req.load("http://premium4.me/api/hosters.php?authcode=%s" % self.authcode)
+ return [x.strip() for x in page.replace("\"", "").split(";")]
\ No newline at end of file diff --git a/module/plugins/accounts/RealdebridCom.py b/module/plugins/accounts/RealdebridCom.py index 3137987a9..4a2cf9368 100644 --- a/module/plugins/accounts/RealdebridCom.py +++ b/module/plugins/accounts/RealdebridCom.py @@ -1,15 +1,18 @@ -from module.plugins.Account import Account
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from module.plugins.MultiHoster import MultiHoster
import xml.dom.minidom as dom
-class RealdebridCom(Account):
+class RealdebridCom(MultiHoster):
__name__ = "RealdebridCom"
- __version__ = "0.4"
+ __version__ = "0.5"
__type__ = "account"
__description__ = """Real-Debrid.com account plugin"""
__author_name__ = ("Devirex, Hazzard")
__author_mail__ = ("naibaf_11@yahoo.de")
- def loadAccountInfo(self, user, req):
+ def loadAccountInfo(self, req):
page = req.load("http://real-debrid.com/api/account.php")
xml = dom.parseString(page)
account_info = {"validuntil": int(xml.getElementsByTagName("expiration")[0].childNodes[0].nodeValue),
@@ -17,9 +20,16 @@ class RealdebridCom(Account): return account_info
- def login(self, user, data, req):
- page = req.load("https://real-debrid.com/ajax/login.php?user=%s&pass=%s" % (user, data["password"]))
+ def login(self, req):
+ page = req.load("https://real-debrid.com/ajax/login.php?user=%s&pass=%s" % (self.loginname, self.password))
#page = req.load("https://real-debrid.com/login.html", post={"user": user, "pass": data["password"]}, cookies=True)
if "Your login informations are incorrect" in page:
self.wrongPassword()
+
+
+ def loadHosterList(self, req):
+ https = "https" if self.getConfig("https") else "http"
+ page = req.load(https + "://real-debrid.com/api/hosters.php").replace("\"","").strip()
+
+ return[x.strip() for x in page.split(",") if x.strip()]
\ No newline at end of file diff --git a/module/plugins/accounts/ShareonlineBiz.py b/module/plugins/accounts/ShareonlineBiz.py index 426f5a6a9..4dd398d6d 100644 --- a/module/plugins/accounts/ShareonlineBiz.py +++ b/module/plugins/accounts/ShareonlineBiz.py @@ -23,24 +23,23 @@ import re class ShareonlineBiz(Account): __name__ = "ShareonlineBiz" - __version__ = "0.21" + __version__ = "0.3" __type__ = "account" __description__ = """share-online.biz account plugin""" __author_name__ = ("mkaay") __author_mail__ = ("mkaay@mkaay.de") - def getUserAPI(self, user, req): - src = req.load("http://api.share-online.biz/account.php?username=%s&password=%s&act=userDetails" % (user, self.accounts[user]["password"])) + def getUserAPI(self, req): + src = req.load("http://api.share-online.biz/account.php?username=%s&password=%s&act=userDetails" % (self.loginname, self.password)) info = {} for line in src.splitlines(): - if "=" in line: - key, value = line.split("=") - info[key] = value + key, value = line.split("=") + info[key] = value return info def loadAccountInfo(self, user, req): try: - info = self.getUserAPI(user, req) + info = self.getUserAPI(req) return {"validuntil": int(info["expire_date"]), "trafficleft": -1, "premium": not info["group"] == "Sammler"} except: pass diff --git a/module/plugins/accounts/ZeveraCom.py b/module/plugins/accounts/ZeveraCom.py new file mode 100644 index 000000000..26eac64b6 --- /dev/null +++ b/module/plugins/accounts/ZeveraCom.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*-
+from module.plugins.MultiHoster import MultiHoster
+
+import re
+from time import mktime, strptime
+
+class ZeveraCom(MultiHoster):
+ __name__ = "ZeveraCom"
+ __version__ = "0.11"
+ __type__ = "account"
+ __description__ = """Zevera.com account plugin"""
+ __author_name__ = ("zoidberg")
+ __author_mail__ = ("zoidberg@mujmail.cz")
+
+ api_url = "http://zevera.com/API.ashx"
+
+ def loadAccountInfo(self, req):
+ dataRet = self.loadAPIRequest(req)
+ account_info = {
+ "trafficleft": dataRet['AccountInfo']['AvailableTODAYTrafficForUseInMBytes'] * 1024,
+ "validuntil": -1 #dataRet['AccountInfo']['EndSubscriptionDate']
+ }
+
+ return account_info
+
+ def login(self, req):
+ if self.loadAPIRequest(req, parse = False) == 'Login Error':
+ self.wrongPassword()
+
+ def loadHosterList(self, req):
+ page = req.load("http://www.zevera.com/jDownloader.ashx?cmd=gethosters")
+ return [x.strip() for x in page.replace("\"", "").split(",")]
+
+ def loadAPIRequest(self, req, parse = True, **kwargs):
+ get_dict = {
+ 'cmd': 'download_request',
+ 'login': self.loginname,
+ 'pass': self.password
+ }
+ get_dict.update(kwargs)
+
+ response = req.load(self.api_url, get = get_dict, decode = True)
+ self.logDebug(response)
+ return self.parseAPIRequest(response) if parse else response
+
+ def parseAPIRequest(self, api_response):
+
+ try:
+ arFields = iter(api_response.split('TAG BEGIN DATA#')[1].split('#END DATA')[0].split('#'))
+
+ retData = {
+ 'VersionMajor': arFields.next(),
+ 'VersionMinor': arFields.next(),
+ 'ErrorCode': int(arFields.next()),
+ 'ErrorMessage': arFields.next(),
+ 'Update_Wait': arFields.next()
+ }
+ serverInfo = {
+ 'DateTimeOnServer': mktime(strptime(arFields.next(),"%Y/%m/%d %H:%M:%S")),
+ 'DAY_Traffic_LimitInMBytes': int(arFields.next())
+ }
+ accountInfo = {
+ 'EndSubscriptionDate': mktime(strptime(arFields.next(),"%Y/%m/%d %H:%M:%S")),
+ 'TrafficUsedInMBytesDayToday': int(arFields.next()),
+ 'AvailableEXTRATrafficForUseInMBytes': int(arFields.next()),
+ 'AvailableTODAYTrafficForUseInMBytes': int(arFields.next())
+ }
+ fileInfo = {
+ 'FileID': arFields.next(),
+ 'Title': arFields.next(),
+ 'RealFileName': arFields.next(),
+ 'FileNameOnServer': arFields.next(),
+ 'StorageServerURL': arFields.next(),
+ 'Token': arFields.next(),
+ 'FileSizeInBytes': int(arFields.next()),
+ 'StatusID': int(arFields.next())
+ }
+ progress = {
+ 'BytesReceived': int(arFields.next()),
+ 'TotalBytesToReceive': int(arFields.next()),
+ 'Percentage': arFields.next(),
+ 'StatusText': arFields.next(),
+ 'ProgressText': arFields.next()
+ }
+ fileInfo.update({
+ 'Progress': progress,
+ 'FilePassword': arFields.next(),
+ 'Keywords': arFields.next(),
+ 'ImageURL4Download': arFields.next(),
+ 'CategoryID': arFields.next(),
+ 'CategoryText': arFields.next(),
+ 'Notes': arFields.next()
+ })
+ retData.update({
+ 'ServerInfo': serverInfo,
+ 'AccountInfo': accountInfo,
+ 'FileInfo': fileInfo
+ })
+
+ except Exception, e:
+ self.logError(e)
+ return None
+
+ self.logDebug(retData)
+ return retData
\ No newline at end of file diff --git a/module/plugins/container/CCF.py b/module/plugins/container/CCF.py index 301b033d4..ab7ff1099 100644 --- a/module/plugins/container/CCF.py +++ b/module/plugins/container/CCF.py @@ -4,13 +4,13 @@ import re from urllib2 import build_opener -from module.plugins.Container import Container +from module.plugins.Crypter import Crypter from module.lib.MultipartPostHandler import MultipartPostHandler from os import makedirs from os.path import exists, join -class CCF(Container): +class CCF(Crypter): __name__ = "CCF" __version__ = "0.2" __pattern__ = r"(?!http://).*\.ccf$" diff --git a/module/plugins/container/RSDF.py b/module/plugins/container/RSDF.py index ea5cd67f2..cbc9864b1 100644 --- a/module/plugins/container/RSDF.py +++ b/module/plugins/container/RSDF.py @@ -5,9 +5,9 @@ import base64 import binascii import re -from module.plugins.Container import Container +from module.plugins.Crypter import Crypter -class RSDF(Container): +class RSDF(Crypter): __name__ = "RSDF" __version__ = "0.21" __pattern__ = r".*\.rsdf" diff --git a/module/plugins/crypter/FilesonicComFolder.py b/module/plugins/crypter/FilesonicComFolder.py index b967a74a1..02ae66295 100644 --- a/module/plugins/crypter/FilesonicComFolder.py +++ b/module/plugins/crypter/FilesonicComFolder.py @@ -4,8 +4,6 @@ import re from module.plugins.Crypter import Crypter class FilesonicComFolder(Crypter): - __name__ = "FilesonicComFolder" - __type__ = "crypter" __pattern__ = r"http://(\w*\.)?(sharingmatrix|filesonic|wupload)\.[^/]*/folder/\w+/?" __version__ = "0.11" __description__ = """Filesonic.com/Wupload.com Folder Plugin""" @@ -15,9 +13,8 @@ class FilesonicComFolder(Crypter): FOLDER_PATTERN = r'<table>\s*<caption>Files Folder</caption>(.*?)</table>' LINK_PATTERN = r'<a href="([^"]+)">' - def decrypt(self, pyfile): - html = self.load(self.pyfile.url) - + def decryptURL(self, url): + html = self.load(url) new_links = [] folder = re.search(self.FOLDER_PATTERN, html, re.DOTALL) @@ -26,6 +23,7 @@ class FilesonicComFolder(Crypter): new_links.extend(re.findall(self.LINK_PATTERN, folder.group(1))) if new_links: - self.core.files.addLinks(new_links, self.pyfile.package().id) + return new_links else: - self.fail('Could not extract any links')
\ No newline at end of file + self.fail('Could not extract any links') + diff --git a/module/plugins/container/LinkList.py b/module/plugins/crypter/LinkList.py index b9eb4b972..ebfa373eb 100644 --- a/module/plugins/container/LinkList.py +++ b/module/plugins/crypter/LinkList.py @@ -1,25 +1,25 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from module.plugins.Crypter import Crypter, Package -from module.plugins.Container import Container - -class LinkList(Container): +class LinkList(Crypter): __name__ = "LinkList" __version__ = "0.11" __pattern__ = r".+\.txt$" __description__ = """Read Link Lists in txt format""" - __config__ = [("clear", "bool", "Clear Linklist after adding", False)] __author_name__ = ("spoob", "jeix") __author_mail__ = ("spoob@pyload.org", "jeix@hasnomail.com") + # method declaration is needed here + def decryptURL(self, url): + return Crypter.decryptURL(self, url) - def decrypt(self, pyfile): - txt = open(pyfile.url, 'r') - links = txt.readlines() - curPack = "Parsed links from %s" % pyfile.name - - packages = {curPack:[],} + def decryptFile(self, content): + links = content.splitlines() + + curPack = "default" + packages = {curPack:[]} for link in links: link = link.strip() @@ -33,10 +33,8 @@ class LinkList(Container): packages[curPack] = [] continue packages[curPack].append(link) - txt.close() # empty packages fix - delete = [] for key,value in packages.iteritems(): @@ -46,12 +44,12 @@ class LinkList(Container): for key in delete: del packages[key] - if self.getConfig("clear"): - try: - txt = open(pyfile.url, 'wb') - txt.close() - except: - self.log.warning(_("LinkList could not be cleared.")) - + urls = [] + for name, links in packages.iteritems(): - self.packages.append((name, links, name)) + if name == "default": + urls.extend(links) + else: + urls.append(Package(name, links)) + + return urls
\ No newline at end of file diff --git a/module/plugins/crypter/XfilesharingProFolder.py b/module/plugins/crypter/XfilesharingProFolder.py new file mode 100644 index 000000000..8e58c207d --- /dev/null +++ b/module/plugins/crypter/XfilesharingProFolder.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*-
+
+from module.plugins.Crypter import Crypter, Package
+import re
+
+class XfilesharingProFolder(Crypter):
+ __name__ = "XfilesharingProFolder"
+ __type__ = "crypter"
+ __pattern__ = r"http://(?:www\.)?((easybytez|turboupload|uploadville|file4safe|fileband|filebeep|grupload|247upload)\.com|(muchshare|annonhost).net|bzlink.us)/users/.*"
+ __version__ = "0.01"
+ __description__ = """Generic XfilesharingPro Folder Plugin"""
+ __author_name__ = ("zoidberg")
+ __author_mail__ = ("zoidberg@mujmail.cz")
+
+ LINK_PATTERN = r'<div class="link"><a href="([^"]+)" target="_blank">[^<]*</a></div>'
+ SUBFOLDER_PATTERN = r'<TD width="1%"><img src="[^"]*/images/folder2.gif"></TD><TD><a href="([^"]+)"><b>(?!\. \.<)([^<]+)</b></a></TD>'
+
+ def decryptURL(self, url):
+ return self.decryptFile(self.load(url, decode = True))
+
+ def decryptFile(self, html):
+ new_links = []
+
+ new_links.extend(re.findall(self.LINK_PATTERN, html))
+
+ subfolders = re.findall(self.SUBFOLDER_PATTERN, html)
+ #self.logDebug(subfolders)
+ for (url, name) in subfolders:
+ if self.package: name = "%s/%s" % (self.package.name, name)
+ new_links.append(Package(name, [url]))
+
+ if not new_links: self.fail('Could not extract any links')
+
+ return new_links
\ No newline at end of file diff --git a/module/plugins/hooks/ClickAndLoad.py b/module/plugins/hooks/ClickAndLoad.py index 97e5cd57d..fc32d0da8 100644 --- a/module/plugins/hooks/ClickAndLoad.py +++ b/module/plugins/hooks/ClickAndLoad.py @@ -32,7 +32,7 @@ class ClickAndLoad(Hook): __author_name__ = ("RaNaN", "mkaay") __author_mail__ = ("RaNaN@pyload.de", "mkaay@mkaay.de") - def coreReady(self): + def activate(self): self.port = int(self.core.config['webinterface']['port']) if self.core.config['webinterface']['activated']: try: diff --git a/module/plugins/hooks/Ev0InFetcher.py b/module/plugins/hooks/Ev0InFetcher.py index 5941cf38c..0cd3f3226 100644 --- a/module/plugins/hooks/Ev0InFetcher.py +++ b/module/plugins/hooks/Ev0InFetcher.py @@ -40,7 +40,7 @@ class Ev0InFetcher(Hook): results = self.core.pluginManager.parseUrls(links) sortedLinks = {} - for url, hoster in results: + for url, hoster in results[0]: if hoster not in sortedLinks: sortedLinks[hoster] = [] sortedLinks[hoster].append(url) diff --git a/module/plugins/hooks/ExternalScripts.py b/module/plugins/hooks/ExternalScripts.py index 2e77f1dae..39fe2b9f0 100644 --- a/module/plugins/hooks/ExternalScripts.py +++ b/module/plugins/hooks/ExternalScripts.py @@ -14,16 +14,15 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - @author: mkaay - @interface-version: 0.1 + @author: RaNaN """ import subprocess -from os import listdir, access, X_OK, makedirs -from os.path import join, exists, basename +from os import access, X_OK, makedirs +from os.path import basename from module.plugins.Hook import Hook -from module.utils import save_join +from module.utils.fs import save_join, exists, join, listdir class ExternalScripts(Hook): __name__ = "ExternalScripts" diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index c789495ca..12bd40d1b 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -3,8 +3,7 @@ import sys import os -from os import remove, chmod, makedirs -from os.path import exists, basename, isfile, isdir, join +from os.path import basename, isfile, isdir, join from traceback import print_exc from copy import copy @@ -49,7 +48,7 @@ if os.name != "nt": from pwd import getpwnam from grp import getgrnam -from module.utils import save_join, fs_encode +from module.utils.fs import save_join, fs_encode, exists, remove, chmod, makedirs from module.plugins.Hook import Hook, threaded, Expose from module.plugins.internal.AbstractExtractor import ArchiveError, CRCError, WrongPassword diff --git a/module/plugins/hooks/MultiHoster.py b/module/plugins/hooks/MultiHoster.py new file mode 100644 index 000000000..749f2c104 --- /dev/null +++ b/module/plugins/hooks/MultiHoster.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +from types import MethodType + +from module.plugins.MultiHoster import MultiHoster as MultiHosterAccount, normalize +from module.plugins.Hook import Hook, AddEventListener +from module.plugins.PluginManager import PluginTuple + +class MultiHoster(Hook): + __version__ = "0.1" + __description__ = "Gives ability to use MultiHoster services. You need to add your account first." + __config__ = [("activated", "bool", "Activated", True)] + __author_mail__ = ("pyLoad Team",) + __author_mail__ = ("support@pyload.org",) + + #TODO: multiple accounts - multihoster / config options + + def init(self): + + # overwritten plugins + self.plugins = {} + + def addHoster(self, account): + + self.logDebug("New MultiHoster %s" % account.__name__) + + pluginMap = {} + for name in self.core.pluginManager.getPlugins("hoster").keys(): + pluginMap[name.lower()] = name + + supported = [] + new_supported = [] + + for hoster in account.getHosterList(): + name = normalize(hoster) + + if name in pluginMap: + supported.append(pluginMap[name]) + else: + new_supported.append(hoster) + + if not supported and not new_supported: + account.logError(_("No Hoster loaded")) + return + + klass = self.core.pluginManager.getPluginClass(account.__name__) + + # inject plugin plugin + account.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(supported))) + for hoster in supported: + self.plugins[hoster] = klass + + account.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) + + # create new regexp + regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) + + # recreate plugin tuple for new regexp + hoster = self.core.pluginManager.getPlugins("hoster") + p = hoster[account.__name__] + new = PluginTuple(p.version, re.compile(regexp), p.deps, p.user, p.path) + hoster[account.__name__] = new + + + + @AddEventListener("accountDeleted") + def refreshAccounts(self, plugin=None, user=None): + + self.plugins = {} + + for name, account in self.core.accountManager.iterAccounts(): + if isinstance(account, MultiHosterAccount) and account.isUsable(): + self.addHoster(account) + + @AddEventListener("accountUpdated") + def refreshAccount(self, plugin, user): + + account = self.core.accountManager.getAccount(plugin, user) + if isinstance(account, MultiHosterAccount) and account.isUsable(): + self.addHoster(account) + + def activate(self): + self.refreshAccounts() + + # new method for plugin manager + def getPlugin(self2, name): + if name in self.plugins: + return self.plugins[name] + return self2.getPluginClass(name) + + pm = self.core.pluginManager + pm.getPlugin = MethodType(getPlugin, pm, object) + + + def deactivate(self): + #restore state + pm = self.core.pluginManager + pm.getPlugin = pm.getPluginClass + diff --git a/module/plugins/hooks/RealdebridCom.py b/module/plugins/hooks/RealdebridCom.py deleted file mode 100644 index c57e3de52..000000000 --- a/module/plugins/hooks/RealdebridCom.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.network.RequestFactory import getURL -from module.plugins.internal.MultiHoster import MultiHoster - -class RealdebridCom(MultiHoster): - __name__ = "RealdebridCom" - __version__ = "0.4" - __type__ = "hook" - - __config__ = [("activated", "bool", "Activated", "False"), - ("https", "bool", "Enable HTTPS", "False")] - - __description__ = """Real-Debrid.com hook plugin""" - __author_name__ = ("Devirex, Hazzard") - __author_mail__ = ("naibaf_11@yahoo.de") - - replacements = [("freakshare.net", "freakshare.com")] - - def getHoster(self): - https = "https" if self.getConfig("https") else "http" - page = getURL(https + "://real-debrid.com/api/hosters.php").replace("\"","").strip() - - return[x.strip() for x in page.split(",") if x.strip()]
\ No newline at end of file diff --git a/module/plugins/hooks/UpdateManager.py b/module/plugins/hooks/UpdateManager.py index ce75399c5..230a6e858 100644 --- a/module/plugins/hooks/UpdateManager.py +++ b/module/plugins/hooks/UpdateManager.py @@ -23,7 +23,7 @@ from os import stat from os.path import join, exists from time import time -from module.ConfigParser import IGNORE +from module.plugins.PluginManager import IGNORE from module.network.RequestFactory import getURL from module.plugins.Hook import threaded, Expose, Hook @@ -60,6 +60,11 @@ class UpdateManager(Hook): @threaded def periodical(self): + + if self.core.version.endswith("-dev"): + self.logDebug("No update check performed on dev version.") + return + update = self.checkForUpdate() if update: self.info["pyload"] = True @@ -129,10 +134,10 @@ class UpdateManager(Hook): else: type = prefix - plugins = getattr(self.core.pluginManager, "%sPlugins" % type) + plugins = self.core.pluginManager.getPlugins(type) if name in plugins: - if float(plugins[name]["v"]) >= float(version): + if float(plugins[name].version) >= float(version): continue if name in IGNORE or (type, name) in IGNORE: diff --git a/module/plugins/hooks/XMPPInterface.py b/module/plugins/hooks/XMPPInterface.py index a96adf524..de87433cf 100644 --- a/module/plugins/hooks/XMPPInterface.py +++ b/module/plugins/hooks/XMPPInterface.py @@ -19,7 +19,7 @@ """ from pyxmpp import streamtls -from pyxmpp.all import JID, Message +from pyxmpp.all import JID, Message, Presence from pyxmpp.jabber.client import JabberClient from pyxmpp.interface import implements from pyxmpp.interfaces import * @@ -121,7 +121,26 @@ class XMPPInterface(IRCInterface, JabberClient): in a client session.""" return [ ("normal", self.message), - ] + ] + + def presence_control(self, stanza): + from_jid = unicode(stanza.get_from_jid()) + stanza_type = stanza.get_type() + self.log.debug("pyLoad XMPP: %s stanza from %s" % (stanza_type, + from_jid)) + + if from_jid in self.getConfig("owners"): + return stanza.make_accept_response() + + return stanza.make_deny_response() + + def session_started(self): + self.stream.send(Presence()) + + self.stream.set_presence_handler("subscribe", self.presence_control) + self.stream.set_presence_handler("subscribed", self.presence_control) + self.stream.set_presence_handler("unsubscribe", self.presence_control) + self.stream.set_presence_handler("unsubscribed", self.presence_control) def message(self, stanza): """Message handler for the component.""" @@ -248,4 +267,10 @@ class VersionHandler(object): q.newTextChild(q.ns(), "name", "Echo component") q.newTextChild(q.ns(), "version", "1.0") return iq -
\ No newline at end of file + + def unload(self): + self.log.debug("pyLoad XMPP: unloading") + self.disconnect() + + def deactivate(self): + self.unload() diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 2de47940d..0e9595265 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -1,12 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from urlparse import urlparse -from re import search -from urllib import unquote from module.network.HTTPRequest import BadHeader from module.plugins.Hoster import Hoster -from module.utils import html_unescape, remove_chars +from module.utils import html_unescape class BasePlugin(Hoster): __name__ = "BasePlugin" @@ -62,28 +60,5 @@ class BasePlugin(Hoster): def downloadFile(self, pyfile): - header = self.load(pyfile.url, just_header = True) - #self.logDebug(header) - - if 'location' in header: - self.logDebug("Location: " + header['location']) - url = unquote(header['location']) - else: - url = pyfile.url - - name = html_unescape(urlparse(url).path.split("/")[-1]) - - if 'content-disposition' in header: - self.logDebug("Content-Disposition: " + header['content-disposition']) - m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", header['content-disposition']) - if m: - disp = m.groupdict() - self.logDebug(disp) - if not disp['enc']: disp['enc'] = 'utf-8' - name = remove_chars(disp['name'], "\"';").strip() - name = unicode(unquote(name), disp['enc']) - - if not name: name = url - pyfile.name = name - self.logDebug("Filename: %s" % pyfile.name) - self.download(url, disposition=True)
\ No newline at end of file + pyfile.name = html_unescape(urlparse(pyfile.url).path.split("/")[-1]) + self.download(pyfile.url, disposition=True) diff --git a/module/plugins/hoster/BezvadataCz.py b/module/plugins/hoster/BezvadataCz.py index 680bbc173..a0717ad64 100644 --- a/module/plugins/hoster/BezvadataCz.py +++ b/module/plugins/hoster/BezvadataCz.py @@ -23,7 +23,7 @@ class BezvadataCz(SimpleHoster): __name__ = "BezvadataCz" __type__ = "hoster" __pattern__ = r"http://(\w*\.)*bezvadata.cz/stahnout/.*" - __version__ = "0.22" + __version__ = "0.23" __description__ = """BezvaData.cz""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") diff --git a/module/plugins/hoster/DlFreeFr.py b/module/plugins/hoster/DlFreeFr.py index 138d0e056..7cb58e6f4 100644 --- a/module/plugins/hoster/DlFreeFr.py +++ b/module/plugins/hoster/DlFreeFr.py @@ -19,7 +19,6 @@ class DlFreeFr(SimpleHoster): FILE_URL_PATTERN = r'href="(?P<url>http://.*?)">Télécharger ce fichier' def setup(self): - self.multiDL = True self.limitDL = 5 self.resumeDownload = True self.chunkLimit = 1 diff --git a/module/plugins/hoster/EasybytezCom.py b/module/plugins/hoster/EasybytezCom.py index 4fbd08756..0b46acb83 100644 --- a/module/plugins/hoster/EasybytezCom.py +++ b/module/plugins/hoster/EasybytezCom.py @@ -23,8 +23,8 @@ from random import random class EasybytezCom(SimpleHoster): __name__ = "EasybytezCom" __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?easybytez.com/(\w+).*" - __version__ = "0.05" + __pattern__ = r"http://(?:\w*\.)?easybytez.com/(\w{6,}).*" + __version__ = "0.06" __description__ = """easybytez.com""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") diff --git a/module/plugins/hoster/EuroshareEu.py b/module/plugins/hoster/EuroshareEu.py index a0bfe0ab2..1e1cc0b4b 100644 --- a/module/plugins/hoster/EuroshareEu.py +++ b/module/plugins/hoster/EuroshareEu.py @@ -37,7 +37,7 @@ class EuroshareEu(Hoster): __name__ = "EuroshareEu" __type__ = "hoster" __pattern__ = r"http://(\w*\.)?euroshare.eu/file/.*" - __version__ = "0.2b" + __version__ = "0.30" __description__ = """Euroshare.eu""" __author_name__ = ("zoidberg") diff --git a/module/plugins/hoster/FilesMailRu.py b/module/plugins/hoster/FilesMailRu.py index 6002ab3dc..1284329b5 100644 --- a/module/plugins/hoster/FilesMailRu.py +++ b/module/plugins/hoster/FilesMailRu.py @@ -2,9 +2,8 @@ # -*- coding: utf-8 -*- import re -from module.plugins.Hoster import Hoster +from module.plugins.Hoster import Hoster, chunks from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks def getInfo(urls): result = [] diff --git a/module/plugins/hoster/FilesonicCom.py b/module/plugins/hoster/FilesonicCom.py index 2788e7c62..f0fbab1d9 100644 --- a/module/plugins/hoster/FilesonicCom.py +++ b/module/plugins/hoster/FilesonicCom.py @@ -7,7 +7,7 @@ from urllib import unquote from module.plugins.Hoster import Hoster
from module.plugins.ReCaptcha import ReCaptcha
-from module.plugins.Plugin import chunks
+from module.utils import chunks
from module.network.RequestFactory import getURL
from module.common.json_layer import json_loads
diff --git a/module/plugins/hoster/MultishareCz.py b/module/plugins/hoster/MultishareCz.py index a0dda30b8..af7aa94cf 100644 --- a/module/plugins/hoster/MultishareCz.py +++ b/module/plugins/hoster/MultishareCz.py @@ -24,7 +24,7 @@ class MultishareCz(SimpleHoster): __name__ = "MultishareCz" __type__ = "hoster" __pattern__ = r"http://(?:\w*\.)?multishare.cz/stahnout/(?P<ID>\d+).*" - __version__ = "0.34" + __version__ = "0.40" __description__ = """MultiShare.cz""" __author_name__ = ("zoidberg") @@ -50,11 +50,12 @@ class MultishareCz(SimpleHoster): self.download("http://www.multishare.cz/html/download_free.php?ID=%s" % self.fileID) def handlePremium(self): - if not self.checkCredit(): + if not self.checkTrafficLeft(): self.logWarning("Not enough credit left to download file") self.resetAccount() self.download("http://www.multishare.cz/html/download_premium.php?ID=%s" % self.fileID) + self.checkTrafficLeft() def handleOverriden(self): if not self.premium: @@ -63,18 +64,13 @@ class MultishareCz(SimpleHoster): self.html = self.load('http://www.multishare.cz/html/mms_ajax.php', post = {"link": self.pyfile.url}, decode = True) self.getFileInfo() - if not self.checkCredit(): + if not self.checkTrafficLeft(): self.fail("Not enough credit left to download file") url = "http://dl%d.mms.multishare.cz/html/mms_process.php" % round(random()*10000*random()) params = {"u_ID" : self.acc_info["u_ID"], "u_hash" : self.acc_info["u_hash"], "link" : self.pyfile.url} self.logDebug(url, params) self.download(url, get = params) - - def checkCredit(self): - self.acc_info = self.account.getAccountInfo(self.user, True) - self.logInfo("User %s has %i MB left" % (self.user, self.acc_info["trafficleft"]/1024)) - - return self.pyfile.size / 1024 <= self.acc_info["trafficleft"] + self.checkTrafficLeft() getInfo = create_getInfo(MultishareCz)
\ No newline at end of file diff --git a/module/plugins/hoster/NetloadIn.py b/module/plugins/hoster/NetloadIn.py index b2bec873d..382328496 100644 --- a/module/plugins/hoster/NetloadIn.py +++ b/module/plugins/hoster/NetloadIn.py @@ -5,11 +5,9 @@ import re from time import sleep, time +from module.utils import chunks from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks - - def getInfo(urls): ## returns list of tupels (name, size (in bytes), status (see FileDatabase), url) diff --git a/module/plugins/hoster/OronCom.py b/module/plugins/hoster/OronCom.py index 2b9f42798..120aa8ff4 100755 --- a/module/plugins/hoster/OronCom.py +++ b/module/plugins/hoster/OronCom.py @@ -15,7 +15,7 @@ def getInfo(urls): result.append((url, 0, 1, url)) continue - m = re.search(OronCom.FILE_INFO_PATTERN, html, re.MULTILINE) + m = re.search(OronCom.FILE_INFO_PATTERN, html) if m: name = m.group(1) hSize = float(m.group(2).replace(",", ".")) @@ -32,11 +32,11 @@ def getInfo(urls): class OronCom(Hoster): __name__ = "OronCom" __type__ = "hoster" - __pattern__ = r"http://(?:www.)?oron.com/(?!folder)\w+" - __version__ = "0.15" - __description__ = "Oron.com Hoster Plugin" + __pattern__ = r"http://(?:www.)?oron.com/" + __version__ = "0.13" + __description__ = "File Hoster: Oron.com" __author_name__ = ("chrox", "DHMH") - __author_mail__ = ("chrox@pyload.org", "webmaster@pcProfil.de") + __author_mail__ = ("chrox@pyload.org", "DHMH@pyload.org") FILE_INFO_PATTERN = r'(?:Filename|Dateiname): <b class="f_arial f_14px">(.*?)</b>\s*<br>\s*(?:Größe|File size): ([0-9,\.]+) (Kb|Mb|Gb)' @@ -129,13 +129,13 @@ class OronCom(Hoster): self.logError("error in parsing site") def handlePremium(self): - info = self.account.getAccountInfo(self.user, True) - self.logDebug("Traffic left: %s" % info['trafficleft']) + self.account.getAccountInfo(True) + self.logDebug("Traffic left: %s" % self.account.trafficleft) self.logDebug("File Size: %d" % int(self.pyfile.size / 1024)) - if int(self.pyfile.size / 1024) > info["trafficleft"]: + if int(self.pyfile.size / 1024) > self.account.trafficleft: self.logInfo(_("Not enough traffic left")) - self.account.empty(self.user) + self.account.empty() self.fail(_("Traffic exceeded")) post_url = "http://oron.com/" + self.file_id @@ -147,3 +147,4 @@ class OronCom(Hoster): self.html = self.load(post_url, post=post_dict, ref=False, decode=True).encode("utf-8") link = re.search('href="(.*?)" class="atitle"', self.html).group(1) self.download(link) + diff --git a/module/plugins/hoster/Premium4Me.py b/module/plugins/hoster/Premium4Me.py index d029b3df1..cd47a9e91 100644 --- a/module/plugins/hoster/Premium4Me.py +++ b/module/plugins/hoster/Premium4Me.py @@ -6,7 +6,7 @@ from module.plugins.Hoster import Hoster class Premium4Me(Hoster):
__name__ = "Premium4Me"
- __version__ = "0.03"
+ __version__ = "0.10"
__type__ = "hoster"
__pattern__ = r"http://premium4.me/.*"
diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py index 0d927c525..a4a72eb53 100644 --- a/module/plugins/hoster/RapidshareCom.py +++ b/module/plugins/hoster/RapidshareCom.py @@ -52,7 +52,7 @@ class RapidshareCom(Hoster): __pattern__ = r"https?://[\w\.]*?rapidshare.com/(?:files/(?P<id>\d*?)/(?P<name>[^?]+)|#!download\|(?:\w+)\|(?P<id_new>\d+)\|(?P<name_new>[^|]+))" __version__ = "1.37" __description__ = """Rapidshare.com Download Hoster""" - __config__ = [["server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None"]] + __config__ = [("server", "Cogent;Deutsche Telekom;Level(3);Level(3) #2;GlobalCrossing;Level(3) #3;Teleglobe;GlobalCrossing #2;TeliaSonera #2;Teleglobe #2;TeliaSonera #3;TeliaSonera", "Preferred Server", "None")] __author_name__ = ("spoob", "RaNaN", "mkaay") __author_mail__ = ("spoob@pyload.org", "ranan@pyload.org", "mkaay@mkaay.de") @@ -132,7 +132,7 @@ class RapidshareCom(Hoster): self.handleFree() def handlePremium(self): - info = self.account.getAccountInfo(self.user, True) + info = self.account.getAccountInfo(True) self.log.debug("%s: Use Premium Account" % self.__name__) url = self.api_data["mirror"] self.download(url, get={"directstart":1}) diff --git a/module/plugins/hoster/RealdebridCom.py b/module/plugins/hoster/RealdebridCom.py index 4aed32614..3a3ac99b0 100644 --- a/module/plugins/hoster/RealdebridCom.py +++ b/module/plugins/hoster/RealdebridCom.py @@ -5,15 +5,15 @@ import re from urllib import quote, unquote
from random import randrange
+from module.utils import encode
from module.plugins.Hoster import Hoster
class RealdebridCom(Hoster):
- __name__ = "RealdebridCom"
- __version__ = "0.42"
- __type__ = "hoster"
-
+ __version__ = "0.41"
__pattern__ = r"https?://.*real-debrid\..*"
__description__ = """Real-Debrid.com hoster plugin"""
+ __config__ = [("https", "bool", _("Enable HTTPS"), False)]
+
__author_name__ = ("Devirex, Hazzard")
__author_mail__ = ("naibaf_11@yahoo.de")
@@ -44,11 +44,14 @@ class RealdebridCom(Hoster): password = self.getPassword().splitlines()
if not password: password = ""
else: password = password[0]
-
- url = "http://real-debrid.com/lib/ajax/generator.php?lang=en&sl=1&link=%s&passwort=%s" % (quote(pyfile.url, ""), password)
+
+ url = "http://real-debrid.com/ajax/deb.php?lang=en&sl=1&link=%s&passwort=%s" % (quote(encode(pyfile.url), ""), password)
page = self.load(url)
error = re.search(r'<span id="generation-error">(.*)</span>', page)
+ generation_ok = re.search(r'<span id="generation-ok"><a href="(.*)">(.*)</a></span>', page)
+ if generation_ok:
+ page = generation_ok.group(1).strip()
if error:
msg = error.group(1).strip()
@@ -59,6 +62,9 @@ class RealdebridCom(Hoster): self.fail(msg)
elif url == 'error':
self.fail("Your IP is most likely blocked. Please contact RealDebrid support")
+ elif page == "File's hoster is in maintenance. Try again later.":
+ self.logWarning(page)
+ self.tempOffline()
else:
new_url = page
diff --git a/module/plugins/hoster/ShareonlineBiz.py b/module/plugins/hoster/ShareonlineBiz.py index d355eeffe..2d1fc8d85 100644 --- a/module/plugins/hoster/ShareonlineBiz.py +++ b/module/plugins/hoster/ShareonlineBiz.py @@ -7,10 +7,8 @@ import hashlib import random from time import sleep -from module.plugins.Hoster import Hoster +from module.plugins.Hoster import Hoster, chunks from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks - def getInfo(urls): api_url_base = "http://api.share-online.biz/linkcheck.php" @@ -52,7 +50,7 @@ class ShareonlineBiz(Hoster): self.multiDL = False self.chunkLimit = 1 - if self.account and self.account.isPremium(self.user): + if self.premium: self.multiDL = True def process(self, pyfile): @@ -60,7 +58,7 @@ class ShareonlineBiz(Hoster): pyfile.name = self.api_data["filename"] pyfile.sync() - if self.account and self.account.isPremium(self.user): + if self.premium: self.handleAPIPremium() #self.handleWebsitePremium() else: @@ -127,7 +125,7 @@ class ShareonlineBiz(Hoster): def handleAPIPremium(self): #should be working better self.resumeDownload = True - info = self.account.getUserAPI(self.user, self.req) + info = self.account.getUserAPI(self.req) if info["dl"].lower() == "not_available": self.fail("DL API error") self.req.cj.setCookie("share-online.biz", "dl", info["dl"]) diff --git a/module/plugins/hoster/TurbouploadCom.py b/module/plugins/hoster/TurbouploadCom.py index 59939d3c7..5fd81fb69 100644 --- a/module/plugins/hoster/TurbouploadCom.py +++ b/module/plugins/hoster/TurbouploadCom.py @@ -23,8 +23,8 @@ from module.plugins.hoster.EasybytezCom import EasybytezCom class TurbouploadCom(EasybytezCom): __name__ = "TurbouploadCom" __type__ = "hoster" - __pattern__ = r"http://(?:\w*\.)?turboupload.com/(\w+).*" - __version__ = "0.01" + __pattern__ = r"http://(?:\w*\.)?turboupload.com/(\w{6,}).*" + __version__ = "0.02" __description__ = """turboupload.com""" __author_name__ = ("zoidberg") __author_mail__ = ("zoidberg@mujmail.cz") diff --git a/module/plugins/hoster/UploadedTo.py b/module/plugins/hoster/UploadedTo.py index 39483cf86..751dcda25 100644 --- a/module/plugins/hoster/UploadedTo.py +++ b/module/plugins/hoster/UploadedTo.py @@ -2,11 +2,10 @@ import re -from module.utils import html_unescape, parseFileSize +from module.utils import html_unescape, parseFileSize, chunks from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL -from module.plugins.Plugin import chunks from module.plugins.ReCaptcha import ReCaptcha key = "bGhGMkllZXByd2VEZnU5Y2NXbHhYVlZ5cEE1bkEzRUw=".decode('base64') diff --git a/module/plugins/hoster/WuploadCom.py b/module/plugins/hoster/WuploadCom.py index 11b61ae59..ffb082cbe 100644 --- a/module/plugins/hoster/WuploadCom.py +++ b/module/plugins/hoster/WuploadCom.py @@ -3,11 +3,12 @@ import re import string +from urllib import unquote from types import MethodType from module.plugins.Hoster import Hoster -from module.plugins.Plugin import chunks +from module.utils import chunks from module.network.RequestFactory import getURL from module.common.json_layer import json_loads @@ -31,7 +32,7 @@ def getInfo(urls): if item["status"] != "AVAILABLE": result.append((None, 0, 1, ids[str(item["id"])])) else: - result.append((item["filename"], item["size"], 2, ids[str(item["id"])])) + result.append((unquote(item["filename"]), item["size"], 2, ids[str(item["id"])])) yield result @@ -47,7 +48,7 @@ class WuploadCom(Hoster): __name__ = "WuploadCom" __type__ = "hoster" __pattern__ = r"http://[\w\.]*?wupload\..*?/file/(([a-z][0-9]+/)?[0-9]+)(/.*)?" - __version__ = "0.1" + __version__ = "0.2" __description__ = """Wupload com""" __author_name__ = ("jeix", "paulking") __author_mail__ = ("jeix@hasnomail.de", "") diff --git a/module/plugins/hoster/YoutubeCom.py b/module/plugins/hoster/YoutubeCom.py index 2b3ea7ed7..b6ea36a3c 100644 --- a/module/plugins/hoster/YoutubeCom.py +++ b/module/plugins/hoster/YoutubeCom.py @@ -75,10 +75,10 @@ class YoutubeCom(Hoster): fmt_dict[fmt] = unquote(url) self.logDebug("Found links: %s" % fmt_dict) - for fmt in fmt_dict.keys(): + for fmt in fmt_dict.keys(): if fmt not in self.formats: - self.logDebug("FMT not supported: %s" % fmt) - del fmt_dict[fmt] + self.logDebug("FMT not supported: %s" % fmt) + del fmt_dict[fmt] allowed = lambda x: self.getConfig(self.formats[x][0]) sel = lambda x: self.formats[x][3] #select quality index diff --git a/module/plugins/hoster/ZeveraCom.py b/module/plugins/hoster/ZeveraCom.py new file mode 100644 index 000000000..d1fa80802 --- /dev/null +++ b/module/plugins/hoster/ZeveraCom.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from module.plugins.Hoster import Hoster
+from module.utils import html_unescape
+from urllib import quote, unquote
+from time import sleep
+
+class ZeveraCom(Hoster):
+ __name__ = "ZeveraCom"
+ __version__ = "0.11"
+ __type__ = "hoster"
+ __pattern__ = r"http://zevera.com/.*"
+ __description__ = """zevera.com hoster plugin"""
+ __author_name__ = ("zoidberg")
+ __author_mail__ = ("zoidberg@mujmail.cz")
+
+ api_url = "http://zevera.com/API.ashx"
+
+ def process(self, pyfile):
+ if not self.account:
+ self.logError(_("Please enter your zevera.com account or deactivate this plugin"))
+ self.fail("No zevera.com account provided")
+
+ self.logDebug("zevera.com: Old URL: %s" % pyfile.url)
+
+ last_size = retries = 0
+ olink = self.pyfile.url #quote(self.pyfile.url.encode('utf_8'))
+
+ for i in range(100):
+ self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_request', olink = olink)
+ self.checkAPIErrors(self.retData)
+
+ if self.retData['FileInfo']['StatusID'] == 100:
+ break
+ elif self.retData['FileInfo']['StatusID'] == 99:
+ self.fail('Failed to initialize download (99)')
+ else:
+ if self.retData['FileInfo']['Progress']['BytesReceived'] <= last_size:
+ if retries >= 6:
+ self.fail('Failed to initialize download (%d)' % self.retData['FileInfo']['StatusID'] )
+ retries += 1
+ else:
+ retries = 0
+
+ last_size = self.retData['FileInfo']['Progress']['BytesReceived']
+
+ pyfile.progress = self.retData['FileInfo']['Progress']['Percentage']
+
+ self.setWait(self.retData['Update_Wait'])
+ self.wait()
+
+ pyfile.progress = 0
+ pyfile.name = self.crazyDecode(self.retData['FileInfo']['RealFileName'])
+ pyfile.size = self.retData['FileInfo']['FileSizeInBytes']
+
+ self.retData = self.account.loadAPIRequest(self.req, cmd = 'download_start', FileID = self.retData['FileInfo']['FileID'])
+ self.checkAPIErrors(self.retData)
+
+ self.download(self.api_url, get = {
+ 'cmd': "open_stream",
+ 'login': self.account.loginname,
+ 'pass': self.account.password,
+ 'FileID': self.retData['FileInfo']['FileID'],
+ 'startBytes': 0
+ }
+ )
+
+ def checkAPIErrors(self, retData):
+ if not retData:
+ self.fail('Unknown API response')
+
+ if retData['ErrorCode']:
+ self.logError(retData['ErrorCode'], retData['ErrorMessage'])
+ self.fail('ERROR: ' + retData['ErrorMessage'])
+
+ if self.pyfile.size / 1024000 > retData['AccountInfo']['AvailableTODAYTrafficForUseInMBytes']:
+ self.logWarning("Not enough data left to download the file")
+
+ def crazyDecode(self, ustring):
+ # accepts decoded ie. unicode string - API response is double-quoted, double-utf8-encoded
+ # no idea what the proper order of calling these functions would be :-/
+ return html_unescape(unquote(unquote(ustring.replace('@DELIMITER@','#'))).encode('raw_unicode_escape').decode('utf-8'))
\ No newline at end of file diff --git a/module/plugins/internal/MultiHoster.py b/module/plugins/internal/MultiHoster.py deleted file mode 100644 index d50df3943..000000000 --- a/module/plugins/internal/MultiHoster.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import re - -from module.utils import remove_chars -from module.plugins.Hook import Hook - -class MultiHoster(Hook): - """ - Generic MultiHoster plugin - """ - - interval = 0 - hosters = [] - replacements = [] - supported = [] - - def getHosterCached(self): - if not self.hosters: - - try: - self.hosters = self.getHoster() - except Exception, e: - self.logError("%s" % str(e)) - return [] - - for rep in self.replacements: - if rep[0] in self.hosters: - self.hosters.remove(rep[0]) - if rep[1] not in self.hosters: - self.hosters.append(rep[1]) - - return self.hosters - - - def getHoster(self): - """Load list of supported hoster - - :return: List of domain names - """ - raise NotImplementedError - - def coreReady(self): - pluginMap = {} - for name in self.core.pluginManager.hosterPlugins.keys(): - pluginMap[name.lower()] = name - - new_supported = [] - - for hoster in self.getHosterCached(): - name = remove_chars(hoster.lower(), "-.") - - if name in pluginMap: - self.supported.append(pluginMap[name]) - else: - new_supported.append(hoster) - - if not self.supported and not new_supported: - self.logError(_("No Hoster loaded")) - return - - module = self.core.pluginManager.getPlugin(self.__name__) - klass = getattr(module, self.__name__) - - # inject plugin plugin - self.logDebug("Overwritten Hosters: %s" % ", ".join(sorted(self.supported))) - for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - dict["new_module"] = module - dict["new_name"] = self.__name__ - - self.logDebug("New Hosters: %s" % ", ".join(sorted(new_supported))) - - # create new regexp - regexp = r".*(%s).*" % "|".join([klass.__pattern__] + [x.replace(".", "\\.") for x in new_supported]) - - dict = self.core.pluginManager.hosterPlugins[self.__name__] - dict["pattern"] = regexp - dict["re"] = re.compile(regexp) - - - def unload(self): - for hoster in self.supported: - dict = self.core.pluginManager.hosterPlugins[hoster] - if "module" in dict: - del dict["module"] - - del dict["new_module"] - del dict["new_name"]
\ No newline at end of file diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index de6215704..a315fbea3 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -19,11 +19,10 @@ import os import re -from os.path import join from glob import glob from subprocess import Popen, PIPE -from module.utils import save_join, decode +from module.utils.fs import save_join, decode, fs_encode from module.plugins.internal.AbstractExtractor import AbtractExtractor, WrongPassword, ArchiveError, CRCError class UnRar(AbtractExtractor): @@ -39,7 +38,7 @@ class UnRar(AbtractExtractor): @staticmethod def checkDeps(): if os.name == "nt": - UnRar.CMD = join(pypath, "UnRAR.exe") + UnRar.CMD = save_join(pypath, "UnRAR.exe") p = Popen([UnRar.CMD], stdout=PIPE, stderr=PIPE) p.communicate() else: @@ -80,7 +79,7 @@ class UnRar(AbtractExtractor): self.password = "" #save the correct password def checkArchive(self): - p = self.call_unrar("l", "-v", self.file) + p = self.call_unrar("l", "-v", fs_encode(self.file)) out, err = p.communicate() if self.re_wrongpwd.search(err): self.passwordProtected = True @@ -102,7 +101,7 @@ class UnRar(AbtractExtractor): def checkPassword(self, password): #at this point we can only verify header protected files if self.headerProtected: - p = self.call_unrar("l", "-v", self.file, password=password) + p = self.call_unrar("l", "-v", fs_encode(self.file), password=password) out, err = p.communicate() if self.re_wrongpwd.search(err): return False @@ -115,7 +114,7 @@ class UnRar(AbtractExtractor): # popen thinks process is still alive (just like pexpect) - very strange behavior # so for now progress can not be determined correctly - p = self.call_unrar(command, self.file, self.out, password=password) + p = self.call_unrar(command, fs_encode(self.file), self.out, password=password) renice(p.pid, self.renice) progress(0) @@ -141,7 +140,7 @@ class UnRar(AbtractExtractor): def listContent(self): command = "vb" if self.fullpath else "lb" - p = self.call_unrar(command, "-v", self.file, password=self.password) + p = self.call_unrar(command, "-v", fs_encode(self.file), password=self.password) out, err = p.communicate() if "Cannot open" in err: @@ -176,7 +175,7 @@ class UnRar(AbtractExtractor): #NOTE: return codes are not reliable, some kind of threading, cleanup whatever issue call = [self.CMD, command] + args + list(xargs) - self.m.logDebug(" ".join(call)) + self.m.logDebug(" ".join([decode(arg) for arg in call])) p = Popen(call, stdout=PIPE, stderr=PIPE) diff --git a/module/remote/socketbackend/ttypes.py b/module/remote/socketbackend/ttypes.py index f8ea121fa..682b2b52a 100644 --- a/module/remote/socketbackend/ttypes.py +++ b/module/remote/socketbackend/ttypes.py @@ -27,10 +27,6 @@ class DownloadStatus: Unknown = 14 Waiting = 5 -class ElementType: - File = 1 - Package = 0 - class Input: BOOL = 4 CHOICE = 6 @@ -49,17 +45,18 @@ class Output: QUESTION = 2 class AccountInfo(BaseObject): - __slots__ = ['validuntil', 'login', 'options', 'valid', 'trafficleft', 'maxtraffic', 'premium', 'type'] + __slots__ = ['plugin', 'loginname', 'valid', 'validuntil', 'trafficleft', 'maxtraffic', 'premium', 'activated', 'options'] - def __init__(self, validuntil=None, login=None, options=None, valid=None, trafficleft=None, maxtraffic=None, premium=None, type=None): - self.validuntil = validuntil - self.login = login - self.options = options + def __init__(self, plugin=None, loginname=None, valid=None, validuntil=None, trafficleft=None, maxtraffic=None, premium=None, activated=None, options=None): + self.plugin = plugin + self.loginname = loginname self.valid = valid + self.validuntil = validuntil self.trafficleft = trafficleft self.maxtraffic = maxtraffic self.premium = premium - self.type = type + self.activated = activated + self.options = options class CaptchaTask(BaseObject): __slots__ = ['tid', 'data', 'type', 'resultType'] @@ -71,22 +68,26 @@ class CaptchaTask(BaseObject): self.resultType = resultType class ConfigItem(BaseObject): - __slots__ = ['name', 'description', 'value', 'type'] + __slots__ = ['name', 'long_name', 'description', 'type', 'default_value', 'value'] - def __init__(self, name=None, description=None, value=None, type=None): + def __init__(self, name=None, long_name=None, description=None, type=None, default_value=None, value=None): self.name = name + self.long_name = long_name self.description = description - self.value = value self.type = type + self.default_value = default_value + self.value = value class ConfigSection(BaseObject): - __slots__ = ['name', 'description', 'items', 'outline'] + __slots__ = ['name', 'long_name', 'description', 'long_description', 'items', 'handler'] - def __init__(self, name=None, description=None, items=None, outline=None): + def __init__(self, name=None, long_name=None, description=None, long_description=None, items=None, handler=None): self.name = name + self.long_name = long_name self.description = description + self.long_description = long_description self.items = items - self.outline = outline + self.handler = handler class DownloadInfo(BaseObject): __slots__ = ['fid', 'name', 'speed', 'eta', 'format_eta', 'bleft', 'size', 'format_size', 'percent', 'status', 'statusmsg', 'format_wait', 'wait_until', 'packageID', 'packageName', 'plugin'] @@ -110,13 +111,11 @@ class DownloadInfo(BaseObject): self.plugin = plugin class EventInfo(BaseObject): - __slots__ = ['eventname', 'id', 'type', 'destination'] + __slots__ = ['eventname', 'event_args'] - def __init__(self, eventname=None, id=None, type=None, destination=None): + def __init__(self, eventname=None, event_args=None): self.eventname = eventname - self.id = id - self.type = type - self.destination = destination + self.event_args = event_args class FileData(BaseObject): __slots__ = ['fid', 'url', 'name', 'plugin', 'size', 'format_size', 'status', 'statusmsg', 'packageID', 'error', 'order'] @@ -208,13 +207,12 @@ class ServerStatus(BaseObject): self.reconnect = reconnect class ServiceCall(BaseObject): - __slots__ = ['plugin', 'func', 'arguments', 'parseArguments'] + __slots__ = ['plugin', 'func', 'arguments'] - def __init__(self, plugin=None, func=None, arguments=None, parseArguments=None): + def __init__(self, plugin=None, func=None, arguments=None): self.plugin = plugin self.func = func self.arguments = arguments - self.parseArguments = parseArguments class ServiceDoesNotExists(Exception): __slots__ = ['plugin', 'func'] @@ -239,10 +237,16 @@ class UserData(BaseObject): self.permission = permission self.templateName = templateName +class UserDoesNotExists(Exception): + __slots__ = ['user'] + + def __init__(self, user=None): + self.user = user + class Iface: def addFiles(self, pid, links): pass - def addPackage(self, name, links, dest): + def addPackage(self, name, links, dest, password): pass def call(self, info): pass @@ -252,6 +256,8 @@ class Iface: pass def checkURLs(self, urls): pass + def configureSection(self, section): + pass def deleteFiles(self, fids): pass def deleteFinished(self): @@ -282,7 +288,7 @@ class Iface: pass def getConfig(self): pass - def getConfigValue(self, category, option, section): + def getConfigValue(self, section, option): pass def getEvents(self, uuid): pass @@ -356,7 +362,7 @@ class Iface: pass def setCaptchaResult(self, tid, result): pass - def setConfigValue(self, category, option, value, section): + def setConfigValue(self, section, option, value): pass def setPackageData(self, pid, data): pass diff --git a/module/remote/thriftbackend/pyload.thrift b/module/remote/thriftbackend/pyload.thrift index 1542e651a..a1b328958 100644 --- a/module/remote/thriftbackend/pyload.thrift +++ b/module/remote/thriftbackend/pyload.thrift @@ -2,7 +2,6 @@ namespace java org.pyload.thrift typedef i32 FileID typedef i32 PackageID -typedef i32 TaskID typedef i32 ResultID typedef i32 InteractionID typedef list<string> LinkList @@ -34,11 +33,6 @@ enum Destination { Queue } -enum ElementType { - Package, - File -} - // types for user interaction // some may only be place holder currently not supported // also all input - output combination are not reasonable, see InteractionManager for further info @@ -93,20 +87,6 @@ struct ServerStatus { 7: bool reconnect } -struct ConfigItem { - 1: string name, - 2: string description, - 3: string value, - 4: string type, -} - -struct ConfigSection { - 1: string name, - 2: string description, - 3: list<ConfigItem> items, - 4: optional string outline -} - struct FileData { 1: FileID fid, 2: string url, @@ -149,6 +129,24 @@ struct InteractionTask { 9: string plugin, } +struct ConfigItem { + 1: string name, + 2: string long_name, + 3: string description, + 4: string type, + 5: string default_value, + 6: string value, +} + +struct ConfigSection { + 1: string name, + 2: string long_name, + 3: string description, + 4: string long_description, + 5: optional list<ConfigItem> items, + 6: optional map<string, InteractionTask> handler, +} + struct CaptchaTask { 1: i16 tid, 2: binary data, @@ -158,9 +156,7 @@ struct CaptchaTask { struct EventInfo { 1: string eventname, - 2: optional i32 id, - 3: optional ElementType type, - 4: optional Destination destination + 2: list<string> event_args, } struct UserData { @@ -172,21 +168,21 @@ struct UserData { } struct AccountInfo { - 1: i64 validuntil, - 2: string login, - 3: map<string, list<string>> options, - 4: bool valid, + 1: PluginName plugin, + 2: string loginname, + 3: bool valid, + 4: i64 validuntil, 5: i64 trafficleft, 6: i64 maxtraffic, 7: bool premium, - 8: string type, + 8: bool activated, + 9: map<string, string> options, } struct ServiceCall { 1: PluginName plugin, 2: string func, - 3: optional list<string> arguments, - 4: optional bool parseArguments, //default False + 3: string arguments, // empty string or json encoded list } struct OnlineStatus { @@ -213,6 +209,10 @@ exception FileDoesNotExists{ 1: FileID fid } +exception UserDoesNotExists{ + 1: string user +} + exception ServiceDoesNotExists{ 1: string plugin 2: string func @@ -225,10 +225,11 @@ exception ServiceException{ service Pyload { //config - string getConfigValue(1: string category, 2: string option, 3: string section), - void setConfigValue(1: string category, 2: string option, 3: string value, 4: string section), + string getConfigValue(1: string section, 2: string option), + void setConfigValue(1: string section, 2: string option, 3: string value), map<string, ConfigSection> getConfig(), - map<string, ConfigSection> getPluginConfig(), + map<PluginName, ConfigSection> getPluginConfig(), + ConfigSection configureSection(1: string section), // server status void pauseServer(), @@ -272,9 +273,9 @@ service Pyload { // downloads - adding/deleting list<PackageID> generateAndAddPackages(1: LinkList links, 2: Destination dest), - PackageID addPackage(1: string name, 2: LinkList links, 3: Destination dest), + PackageID addPackage(1: string name, 2: LinkList links, 3: Destination dest, 4: string password), + PackageID uploadContainer(1: string filename, 2: binary data), void addFiles(1: PackageID pid, 2: LinkList links), - void uploadContainer(1: string filename, 2: binary data), void deleteFiles(1: list<FileID> fids), void deletePackages(1: list<PackageID> pids), @@ -306,7 +307,7 @@ service Pyload { //auth bool login(1: string username, 2: string password), - UserData getUserData(1: string username, 2:string password), + UserData getUserData(1: string username, 2:string password) throws (1: UserDoesNotExists ex), map<string, UserData> getAllUserData(), //services @@ -332,6 +333,6 @@ service Pyload { //captcha bool isCaptchaWaiting(), CaptchaTask getCaptchaTask(1: bool exclusive), - string getCaptchaTaskStatus(1: TaskID tid), - void setCaptchaResult(1: TaskID tid, 2: string result), + string getCaptchaTaskStatus(1: InteractionID tid), + void setCaptchaResult(1: InteractionID tid, 2: string result), } diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote index bfaf5b078..6ee40092d 100755 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote @@ -23,10 +23,11 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print 'Usage: ' + sys.argv[0] + ' [-h host[:port]] [-u url] [-f[ramed]] function [arg1 [arg2...]]' print '' print 'Functions:' - print ' string getConfigValue(string category, string option, string section)' - print ' void setConfigValue(string category, string option, string value, string section)' + print ' string getConfigValue(string section, string option)' + print ' void setConfigValue(string section, string option, string value)' print ' getConfig()' print ' getPluginConfig()' + print ' ConfigSection configureSection(string section)' print ' void pauseServer()' print ' void unpauseServer()' print ' bool togglePause()' @@ -56,7 +57,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print ' getPackageOrder(Destination destination)' print ' getFileOrder(PackageID pid)' print ' generateAndAddPackages(LinkList links, Destination dest)' - print ' PackageID addPackage(string name, LinkList links, Destination dest)' + print ' PackageID addPackage(string name, LinkList links, Destination dest, string password)' print ' void addFiles(PackageID pid, LinkList links)' print ' void uploadContainer(string filename, string data)' print ' void deleteFiles( fids)' @@ -145,16 +146,16 @@ client = Pyload.Client(protocol) transport.open() if cmd == 'getConfigValue': - if len(args) != 3: - print 'getConfigValue requires 3 args' + if len(args) != 2: + print 'getConfigValue requires 2 args' sys.exit(1) - pp.pprint(client.getConfigValue(args[0],args[1],args[2],)) + pp.pprint(client.getConfigValue(args[0],args[1],)) elif cmd == 'setConfigValue': - if len(args) != 4: - print 'setConfigValue requires 4 args' + if len(args) != 3: + print 'setConfigValue requires 3 args' sys.exit(1) - pp.pprint(client.setConfigValue(args[0],args[1],args[2],args[3],)) + pp.pprint(client.setConfigValue(args[0],args[1],args[2],)) elif cmd == 'getConfig': if len(args) != 0: @@ -168,6 +169,12 @@ elif cmd == 'getPluginConfig': sys.exit(1) pp.pprint(client.getPluginConfig()) +elif cmd == 'configureSection': + if len(args) != 1: + print 'configureSection requires 1 args' + sys.exit(1) + pp.pprint(client.configureSection(args[0],)) + elif cmd == 'pauseServer': if len(args) != 0: print 'pauseServer requires 0 args' @@ -343,10 +350,10 @@ elif cmd == 'generateAndAddPackages': pp.pprint(client.generateAndAddPackages(eval(args[0]),eval(args[1]),)) elif cmd == 'addPackage': - if len(args) != 3: - print 'addPackage requires 3 args' + if len(args) != 4: + print 'addPackage requires 4 args' sys.exit(1) - pp.pprint(client.addPackage(args[0],eval(args[1]),eval(args[2]),)) + pp.pprint(client.addPackage(args[0],eval(args[1]),eval(args[2]),args[3],)) elif cmd == 'addFiles': if len(args) != 2: diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py index 78a42f16a..3e0fe3bbc 100644 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py @@ -9,26 +9,24 @@ from thrift.Thrift import TType, TMessageType, TException from ttypes import * from thrift.Thrift import TProcessor -from thrift.protocol.TBase import TBase, TExceptionBase +from thrift.protocol.TBase import TBase, TExceptionBase, TApplicationException class Iface(object): - def getConfigValue(self, category, option, section): + def getConfigValue(self, section, option): """ Parameters: - - category - - option - section + - option """ pass - def setConfigValue(self, category, option, value, section): + def setConfigValue(self, section, option, value): """ Parameters: - - category + - section - option - value - - section """ pass @@ -38,6 +36,13 @@ class Iface(object): def getPluginConfig(self, ): pass + def configureSection(self, section): + """ + Parameters: + - section + """ + pass + def pauseServer(self, ): pass @@ -181,12 +186,13 @@ class Iface(object): """ pass - def addPackage(self, name, links, dest): + def addPackage(self, name, links, dest, password): """ Parameters: - name - links - dest + - password """ pass @@ -434,22 +440,20 @@ class Client(Iface): self._oprot = oprot self._seqid = 0 - def getConfigValue(self, category, option, section): + def getConfigValue(self, section, option): """ Parameters: - - category - - option - section + - option """ - self.send_getConfigValue(category, option, section) + self.send_getConfigValue(section, option) return self.recv_getConfigValue() - def send_getConfigValue(self, category, option, section): + def send_getConfigValue(self, section, option): self._oprot.writeMessageBegin('getConfigValue', TMessageType.CALL, self._seqid) args = getConfigValue_args() - args.category = category - args.option = option args.section = section + args.option = option args.write(self._oprot) self._oprot.writeMessageEnd() self._oprot.trans.flush() @@ -468,24 +472,22 @@ class Client(Iface): return result.success raise TApplicationException(TApplicationException.MISSING_RESULT, "getConfigValue failed: unknown result"); - def setConfigValue(self, category, option, value, section): + def setConfigValue(self, section, option, value): """ Parameters: - - category + - section - option - value - - section """ - self.send_setConfigValue(category, option, value, section) + self.send_setConfigValue(section, option, value) self.recv_setConfigValue() - def send_setConfigValue(self, category, option, value, section): + def send_setConfigValue(self, section, option, value): self._oprot.writeMessageBegin('setConfigValue', TMessageType.CALL, self._seqid) args = setConfigValue_args() - args.category = category + args.section = section args.option = option args.value = value - args.section = section args.write(self._oprot) self._oprot.writeMessageEnd() self._oprot.trans.flush() @@ -552,6 +554,36 @@ class Client(Iface): return result.success raise TApplicationException(TApplicationException.MISSING_RESULT, "getPluginConfig failed: unknown result"); + def configureSection(self, section): + """ + Parameters: + - section + """ + self.send_configureSection(section) + return self.recv_configureSection() + + def send_configureSection(self, section): + self._oprot.writeMessageBegin('configureSection', TMessageType.CALL, self._seqid) + args = configureSection_args() + args.section = section + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_configureSection(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = configureSection_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "configureSection failed: unknown result"); + def pauseServer(self, ): self.send_pauseServer() self.recv_pauseServer() @@ -1348,22 +1380,24 @@ class Client(Iface): return result.success raise TApplicationException(TApplicationException.MISSING_RESULT, "generateAndAddPackages failed: unknown result"); - def addPackage(self, name, links, dest): + def addPackage(self, name, links, dest, password): """ Parameters: - name - links - dest + - password """ - self.send_addPackage(name, links, dest) + self.send_addPackage(name, links, dest, password) return self.recv_addPackage() - def send_addPackage(self, name, links, dest): + def send_addPackage(self, name, links, dest, password): self._oprot.writeMessageBegin('addPackage', TMessageType.CALL, self._seqid) args = addPackage_args() args.name = name args.links = links args.dest = dest + args.password = password args.write(self._oprot) self._oprot.writeMessageEnd() self._oprot.trans.flush() @@ -2130,6 +2164,8 @@ class Client(Iface): self._iprot.readMessageEnd() if result.success is not None: return result.success + if result.ex is not None: + raise result.ex raise TApplicationException(TApplicationException.MISSING_RESULT, "getUserData failed: unknown result"); def getAllUserData(self, ): @@ -2427,6 +2463,7 @@ class Processor(Iface, TProcessor): self._processMap["setConfigValue"] = Processor.process_setConfigValue self._processMap["getConfig"] = Processor.process_getConfig self._processMap["getPluginConfig"] = Processor.process_getPluginConfig + self._processMap["configureSection"] = Processor.process_configureSection self._processMap["pauseServer"] = Processor.process_pauseServer self._processMap["unpauseServer"] = Processor.process_unpauseServer self._processMap["togglePause"] = Processor.process_togglePause @@ -2514,7 +2551,7 @@ class Processor(Iface, TProcessor): args.read(iprot) iprot.readMessageEnd() result = getConfigValue_result() - result.success = self._handler.getConfigValue(args.category, args.option, args.section) + result.success = self._handler.getConfigValue(args.section, args.option) oprot.writeMessageBegin("getConfigValue", TMessageType.REPLY, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -2525,7 +2562,7 @@ class Processor(Iface, TProcessor): args.read(iprot) iprot.readMessageEnd() result = setConfigValue_result() - self._handler.setConfigValue(args.category, args.option, args.value, args.section) + self._handler.setConfigValue(args.section, args.option, args.value) oprot.writeMessageBegin("setConfigValue", TMessageType.REPLY, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -2553,6 +2590,17 @@ class Processor(Iface, TProcessor): oprot.writeMessageEnd() oprot.trans.flush() + def process_configureSection(self, seqid, iprot, oprot): + args = configureSection_args() + args.read(iprot) + iprot.readMessageEnd() + result = configureSection_result() + result.success = self._handler.configureSection(args.section) + oprot.writeMessageBegin("configureSection", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_pauseServer(self, seqid, iprot, oprot): args = pauseServer_args() args.read(iprot) @@ -2886,7 +2934,7 @@ class Processor(Iface, TProcessor): args.read(iprot) iprot.readMessageEnd() result = addPackage_result() - result.success = self._handler.addPackage(args.name, args.links, args.dest) + result.success = self._handler.addPackage(args.name, args.links, args.dest, args.password) oprot.writeMessageBegin("addPackage", TMessageType.REPLY, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -3175,7 +3223,10 @@ class Processor(Iface, TProcessor): args.read(iprot) iprot.readMessageEnd() result = getUserData_result() - result.success = self._handler.getUserData(args.username, args.password) + try: + result.success = self._handler.getUserData(args.username, args.password) + except UserDoesNotExists, ex: + result.ex = ex oprot.writeMessageBegin("getUserData", TMessageType.REPLY, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -3302,28 +3353,24 @@ class Processor(Iface, TProcessor): class getConfigValue_args(TBase): """ Attributes: - - category - - option - section + - option """ __slots__ = [ - 'category', - 'option', 'section', + 'option', ] thrift_spec = ( None, # 0 - (1, TType.STRING, 'category', None, None, ), # 1 + (1, TType.STRING, 'section', None, None, ), # 1 (2, TType.STRING, 'option', None, None, ), # 2 - (3, TType.STRING, 'section', None, None, ), # 3 ) - def __init__(self, category=None, option=None, section=None,): - self.category = category - self.option = option + def __init__(self, section=None, option=None,): self.section = section + self.option = option class getConfigValue_result(TBase): @@ -3347,32 +3394,28 @@ class getConfigValue_result(TBase): class setConfigValue_args(TBase): """ Attributes: - - category + - section - option - value - - section """ __slots__ = [ - 'category', + 'section', 'option', 'value', - 'section', ] thrift_spec = ( None, # 0 - (1, TType.STRING, 'category', None, None, ), # 1 + (1, TType.STRING, 'section', None, None, ), # 1 (2, TType.STRING, 'option', None, None, ), # 2 (3, TType.STRING, 'value', None, None, ), # 3 - (4, TType.STRING, 'section', None, None, ), # 4 ) - def __init__(self, category=None, option=None, value=None, section=None,): - self.category = category + def __init__(self, section=None, option=None, value=None,): + self.section = section self.option = option self.value = value - self.section = section class setConfigValue_result(TBase): @@ -3438,6 +3481,43 @@ class getPluginConfig_result(TBase): self.success = success +class configureSection_args(TBase): + """ + Attributes: + - section + """ + + __slots__ = [ + 'section', + ] + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'section', None, None, ), # 1 + ) + + def __init__(self, section=None,): + self.section = section + + +class configureSection_result(TBase): + """ + Attributes: + - success + """ + + __slots__ = [ + 'success', + ] + + thrift_spec = ( + (0, TType.STRUCT, 'success', (ConfigSection, ConfigSection.thrift_spec), None, ), # 0 + ) + + def __init__(self, success=None,): + self.success = success + + class pauseServer_args(TBase): __slots__ = [ @@ -4349,12 +4429,14 @@ class addPackage_args(TBase): - name - links - dest + - password """ __slots__ = [ 'name', 'links', 'dest', + 'password', ] thrift_spec = ( @@ -4362,12 +4444,14 @@ class addPackage_args(TBase): (1, TType.STRING, 'name', None, None, ), # 1 (2, TType.LIST, 'links', (TType.STRING,None), None, ), # 2 (3, TType.I32, 'dest', None, None, ), # 3 + (4, TType.STRING, 'password', None, None, ), # 4 ) - def __init__(self, name=None, links=None, dest=None,): + def __init__(self, name=None, links=None, dest=None, password=None,): self.name = name self.links = links self.dest = dest + self.password = password class addPackage_result(TBase): @@ -5182,18 +5266,22 @@ class getUserData_result(TBase): """ Attributes: - success + - ex """ __slots__ = [ 'success', + 'ex', ] thrift_spec = ( (0, TType.STRUCT, 'success', (UserData, UserData.thrift_spec), None, ), # 0 + (1, TType.STRUCT, 'ex', (UserDoesNotExists, UserDoesNotExists.thrift_spec), None, ), # 1 ) - def __init__(self, success=None,): + def __init__(self, success=None, ex=None,): self.success = success + self.ex = ex class getAllUserData_args(TBase): diff --git a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py index 1299b515d..b2da9748d 100644 --- a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py +++ b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py @@ -78,20 +78,6 @@ class Destination(TBase): "Queue": 1, } -class ElementType(TBase): - Package = 0 - File = 1 - - _VALUES_TO_NAMES = { - 0: "Package", - 1: "File", - } - - _NAMES_TO_VALUES = { - "Package": 0, - "File": 1, - } - class Input(TBase): NONE = 0 TEXT = 1 @@ -270,68 +256,6 @@ class ServerStatus(TBase): self.reconnect = reconnect -class ConfigItem(TBase): - """ - Attributes: - - name - - description - - value - - type - """ - - __slots__ = [ - 'name', - 'description', - 'value', - 'type', - ] - - thrift_spec = ( - None, # 0 - (1, TType.STRING, 'name', None, None, ), # 1 - (2, TType.STRING, 'description', None, None, ), # 2 - (3, TType.STRING, 'value', None, None, ), # 3 - (4, TType.STRING, 'type', None, None, ), # 4 - ) - - def __init__(self, name=None, description=None, value=None, type=None,): - self.name = name - self.description = description - self.value = value - self.type = type - - -class ConfigSection(TBase): - """ - Attributes: - - name - - description - - items - - outline - """ - - __slots__ = [ - 'name', - 'description', - 'items', - 'outline', - ] - - thrift_spec = ( - None, # 0 - (1, TType.STRING, 'name', None, None, ), # 1 - (2, TType.STRING, 'description', None, None, ), # 2 - (3, TType.LIST, 'items', (TType.STRUCT,(ConfigItem, ConfigItem.thrift_spec)), None, ), # 3 - (4, TType.STRING, 'outline', None, None, ), # 4 - ) - - def __init__(self, name=None, description=None, items=None, outline=None,): - self.name = name - self.description = description - self.items = items - self.outline = outline - - class FileData(TBase): """ Attributes: @@ -509,6 +433,84 @@ class InteractionTask(TBase): self.plugin = plugin +class ConfigItem(TBase): + """ + Attributes: + - name + - long_name + - description + - type + - default_value + - value + """ + + __slots__ = [ + 'name', + 'long_name', + 'description', + 'type', + 'default_value', + 'value', + ] + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'name', None, None, ), # 1 + (2, TType.STRING, 'long_name', None, None, ), # 2 + (3, TType.STRING, 'description', None, None, ), # 3 + (4, TType.STRING, 'type', None, None, ), # 4 + (5, TType.STRING, 'default_value', None, None, ), # 5 + (6, TType.STRING, 'value', None, None, ), # 6 + ) + + def __init__(self, name=None, long_name=None, description=None, type=None, default_value=None, value=None,): + self.name = name + self.long_name = long_name + self.description = description + self.type = type + self.default_value = default_value + self.value = value + + +class ConfigSection(TBase): + """ + Attributes: + - name + - long_name + - description + - long_description + - items + - handler + """ + + __slots__ = [ + 'name', + 'long_name', + 'description', + 'long_description', + 'items', + 'handler', + ] + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'name', None, None, ), # 1 + (2, TType.STRING, 'long_name', None, None, ), # 2 + (3, TType.STRING, 'description', None, None, ), # 3 + (4, TType.STRING, 'long_description', None, None, ), # 4 + (5, TType.LIST, 'items', (TType.STRUCT,(ConfigItem, ConfigItem.thrift_spec)), None, ), # 5 + (6, TType.MAP, 'handler', (TType.STRING,None,TType.STRUCT,(InteractionTask, InteractionTask.thrift_spec)), None, ), # 6 + ) + + def __init__(self, name=None, long_name=None, description=None, long_description=None, items=None, handler=None,): + self.name = name + self.long_name = long_name + self.description = description + self.long_description = long_description + self.items = items + self.handler = handler + + class CaptchaTask(TBase): """ Attributes: @@ -544,31 +546,23 @@ class EventInfo(TBase): """ Attributes: - eventname - - id - - type - - destination + - event_args """ __slots__ = [ 'eventname', - 'id', - 'type', - 'destination', + 'event_args', ] thrift_spec = ( None, # 0 (1, TType.STRING, 'eventname', None, None, ), # 1 - (2, TType.I32, 'id', None, None, ), # 2 - (3, TType.I32, 'type', None, None, ), # 3 - (4, TType.I32, 'destination', None, None, ), # 4 + (2, TType.LIST, 'event_args', (TType.STRING,None), None, ), # 2 ) - def __init__(self, eventname=None, id=None, type=None, destination=None,): + def __init__(self, eventname=None, event_args=None,): self.eventname = eventname - self.id = id - self.type = type - self.destination = destination + self.event_args = event_args class UserData(TBase): @@ -609,48 +603,52 @@ class UserData(TBase): class AccountInfo(TBase): """ Attributes: - - validuntil - - login - - options + - plugin + - loginname - valid + - validuntil - trafficleft - maxtraffic - premium - - type + - activated + - options """ __slots__ = [ - 'validuntil', - 'login', - 'options', + 'plugin', + 'loginname', 'valid', + 'validuntil', 'trafficleft', 'maxtraffic', 'premium', - 'type', + 'activated', + 'options', ] thrift_spec = ( None, # 0 - (1, TType.I64, 'validuntil', None, None, ), # 1 - (2, TType.STRING, 'login', None, None, ), # 2 - (3, TType.MAP, 'options', (TType.STRING,None,TType.LIST,(TType.STRING,None)), None, ), # 3 - (4, TType.BOOL, 'valid', None, None, ), # 4 + (1, TType.STRING, 'plugin', None, None, ), # 1 + (2, TType.STRING, 'loginname', None, None, ), # 2 + (3, TType.BOOL, 'valid', None, None, ), # 3 + (4, TType.I64, 'validuntil', None, None, ), # 4 (5, TType.I64, 'trafficleft', None, None, ), # 5 (6, TType.I64, 'maxtraffic', None, None, ), # 6 (7, TType.BOOL, 'premium', None, None, ), # 7 - (8, TType.STRING, 'type', None, None, ), # 8 + (8, TType.BOOL, 'activated', None, None, ), # 8 + (9, TType.MAP, 'options', (TType.STRING,None,TType.LIST,(TType.STRING,None)), None, ), # 9 ) - def __init__(self, validuntil=None, login=None, options=None, valid=None, trafficleft=None, maxtraffic=None, premium=None, type=None,): - self.validuntil = validuntil - self.login = login - self.options = options + def __init__(self, plugin=None, loginname=None, valid=None, validuntil=None, trafficleft=None, maxtraffic=None, premium=None, activated=None, options=None,): + self.plugin = plugin + self.loginname = loginname self.valid = valid + self.validuntil = validuntil self.trafficleft = trafficleft self.maxtraffic = maxtraffic self.premium = premium - self.type = type + self.activated = activated + self.options = options class ServiceCall(TBase): @@ -659,29 +657,25 @@ class ServiceCall(TBase): - plugin - func - arguments - - parseArguments """ __slots__ = [ 'plugin', 'func', 'arguments', - 'parseArguments', ] thrift_spec = ( None, # 0 (1, TType.STRING, 'plugin', None, None, ), # 1 (2, TType.STRING, 'func', None, None, ), # 2 - (3, TType.LIST, 'arguments', (TType.STRING,None), None, ), # 3 - (4, TType.BOOL, 'parseArguments', None, None, ), # 4 + (3, TType.STRING, 'arguments', None, None, ), # 3 ) - def __init__(self, plugin=None, func=None, arguments=None, parseArguments=None,): + def __init__(self, plugin=None, func=None, arguments=None,): self.plugin = plugin self.func = func self.arguments = arguments - self.parseArguments = parseArguments class OnlineStatus(TBase): @@ -786,6 +780,28 @@ class FileDoesNotExists(TExceptionBase): return repr(self) +class UserDoesNotExists(TExceptionBase): + """ + Attributes: + - user + """ + + __slots__ = [ + 'user', + ] + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'user', None, None, ), # 1 + ) + + def __init__(self, user=None,): + self.user = user + + def __str__(self): + return repr(self) + + class ServiceDoesNotExists(TExceptionBase): """ Attributes: diff --git a/module/setup.py b/module/setup.py index f90afe23a..d16b8c9e2 100644 --- a/module/setup.py +++ b/module/setup.py @@ -41,15 +41,15 @@ class Setup(): self.stdin_encoding = get_console_encoding(sys.stdin.encoding) def start(self): - langs = self.config.getMetaData("general", "language")["type"].split(";") + langs = self.config.getMetaData("general", "language").type.split(";") lang = self.ask(u"Choose your Language / Wähle deine Sprache", "en", langs) gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) translation = gettext.translation("setup", join(self.path, "locale"), languages=[lang, "en"], fallback=True) translation.install(True) - #Input shorthand for yes + #l10n Input shorthand for yes self.yes = _("y") - #Input shorthand for no + #l10n Input shorthand for no self.no = _("n") # print "" @@ -82,7 +82,7 @@ class Setup(): print _("When you are ready for system check, hit enter.") raw_input() - basic, ssl, captcha, gui, web, js = self.system_check() + basic, ssl, captcha, web, js = self.system_check() print "" if not basic: @@ -101,7 +101,6 @@ class Setup(): if self.check_module("Crypto"): avail.append(_("container decrypting")) if ssl: avail.append(_("ssl connection")) if captcha: avail.append(_("automatic captcha decryption")) - if gui: avail.append(_("GUI")) if web: avail.append(_("Webinterface")) if js: avail.append(_("extended Click'N'Load")) @@ -133,11 +132,6 @@ class Setup(): print _("Only needed for some hosters and as freeuser.") print "" - if not gui: - print _("Gui not available") - print _("The Graphical User Interface.") - print "" - if not js: print _("no JavaScript engine found") print _("You will need this for some Click'N'Load links. Install Spidermonkey, ossp-js, pyv8 or rhino") @@ -232,10 +226,6 @@ class Setup(): print "" - gui = self.check_module("PyQt4") - self.print_dep("PyQt4", gui) - - print "" jinja = True try: @@ -263,7 +253,7 @@ class Setup(): js = True if JsEngine.ENGINE else False self.print_dep(_("JS engine"), js) - return basic, ssl, captcha, gui, web, js + return basic, ssl, captcha, web, js def conf_basic(self): print "" @@ -288,7 +278,7 @@ class Setup(): print "" langs = self.config.getMetaData("general", "language") - self.config["general"]["language"] = self.ask(_("Language"), "en", langs["type"].split(";")) + self.config["general"]["language"] = self.ask(_("Language"), "en", langs.type.split(";")) self.config["general"]["download_folder"] = self.ask(_("Downloadfolder"), "Downloads") self.config["download"]["max_downloads"] = self.ask(_("Max parallel downloads"), "3") @@ -496,10 +486,10 @@ class Setup(): input = default if bool: - # yes, true,t are inputs for booleans with value true + #l10n yes, true,t are inputs for booleans with value true if input.lower().strip() in [self.yes, _("yes"), _("true"), _("t"), "yes"]: return True - # no, false,f are inputs for booleans with value false + #l10n no, false,f are inputs for booleans with value false elif input.lower().strip() in [self.no, _("no"), _("false"), _("f"), "no"]: return False else: diff --git a/module/threads/BaseThread.py b/module/threads/BaseThread.py new file mode 100644 index 000000000..f6fac46a0 --- /dev/null +++ b/module/threads/BaseThread.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import sys +import locale + +from threading import Thread +from time import strftime, gmtime +from sys import exc_info +from types import MethodType +from pprint import pformat +from traceback import format_exc + +from module.utils.fs import listdir, join, save_join, stat, exists + +class BaseThread(Thread): + """abstract base class for thread types""" + + def __init__(self, manager): + Thread.__init__(self) + self.setDaemon(True) + self.m = manager #thread manager + self.core = manager.core + self.log = manager.core.log + + def writeDebugReport(self, name, pyfile=None, plugin=None): + """ writes a debug report to disk """ + + dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S")) + if pyfile: + dump = self.getFileDump(pyfile) + else: + dump = self.getPluginDump(plugin) + + try: + import zipfile + + zip = zipfile.ZipFile(dump_name, "w") + + if exists(join("tmp", name)): + for f in listdir(join("tmp", name)): + try: + # avoid encoding errors + zip.write(join("tmp", name, f), save_join(name, f)) + except: + pass + + info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime()) + info.external_attr = 0644 << 16L # change permissions + zip.writestr(info, dump) + + info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime()) + info.external_attr = 0644 << 16L + zip.writestr(info, self.getSystemDump()) + + zip.close() + + if not stat(dump_name).st_size: + raise Exception("Empty Zipfile") + + except Exception, e: + self.log.debug("Error creating zip file: %s" % e) + + dump_name = dump_name.replace(".zip", ".txt") + f = open(dump_name, "wb") + f.write(dump) + f.close() + + self.log.info("Debug Report written to %s" % dump_name) + return dump_name + + def getFileDump(self, pyfile): + dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( + self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version__, format_exc()) + + tb = exc_info()[2] + stack = [] + while tb: + stack.append(tb.tb_frame) + tb = tb.tb_next + + for frame in stack[1:]: + dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_lineno) + + for key, value in frame.f_locals.items(): + dump += "\t%20s = " % key + try: + dump += pformat(value) + "\n" + except Exception, e: + dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" + + del frame + + del stack #delete it just to be sure... + + dump += "\n\nPLUGIN OBJECT DUMP: \n\n" + + for name in dir(pyfile.plugin): + attr = getattr(pyfile.plugin, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" + + dump += "\nPYFILE OBJECT DUMP: \n\n" + + for name in dir(pyfile): + attr = getattr(pyfile, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" + + dump += "\n\nCONFIG: \n\n" + dump += pformat(self.m.core.config.values) + "\n" + + return dump + + #TODO + def getPluginDump(self, plugin): + return "" + + def getSystemDump(self): + return "" + + def clean(self, pyfile): + """ set thread unactive and release pyfile """ + self.active = False + pyfile.release() diff --git a/module/threads/DecrypterThread.py b/module/threads/DecrypterThread.py new file mode 100644 index 000000000..ce3c8cd83 --- /dev/null +++ b/module/threads/DecrypterThread.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from time import sleep +from traceback import print_exc + +from module.utils import uniqify +from module.plugins.Base import Retry +from module.plugins.Crypter import Package + +from BaseThread import BaseThread + +class DecrypterThread(BaseThread): + """thread for decrypting""" + + def __init__(self, manager, data, pid): + """constructor""" + BaseThread.__init__(self, manager) + self.data = data + self.pid = pid + + self.start() + + def run(self): + plugin_map = {} + for url, plugin in self.data: + if plugin in plugin_map: + plugin_map[plugin].append(url) + else: + plugin_map[plugin] = [url] + + self.decrypt(plugin_map) + + def decrypt(self, plugin_map): + pack = self.m.core.files.getPackage(self.pid) + result = [] + + for name, urls in plugin_map.iteritems(): + klass = self.m.core.pluginManager.loadClass("crypter", name) + plugin = klass(self.m.core, pack, pack.password) + plugin_result = [] + + try: + try: + plugin_result = plugin._decrypt(urls) + except Retry: + sleep(1) + plugin_result = plugin._decrypt(urls) + except Exception, e: + plugin.logError(_("Decrypting failed"), e) + if self.m.core.debug: + print_exc() + self.writeDebugReport(plugin.__name__, plugin=plugin) + + plugin.logDebug("Decrypted", plugin_result) + result.extend(plugin_result) + + result = uniqify(result) + pack_names = {} + urls = [] + + for p in result: + if isinstance(p, Package): + if p.name in pack_names: + pack_names[p.name].urls.extend(p.urls) + else: + pack_names[p.name] = p + else: + urls.append(p) + + if urls: + self.log.info(_("Decrypted %(count)d links into package %(name)s") % {"count": len(urls), "name": pack.name}) + self.m.core.api.addFiles(self.pid, urls) + + for p in pack_names.itervalues(): + self.m.core.api.addPackage(p.name, p.urls, p.dest, pack.password) + + if not result: + self.log.info(_("No links decrypted")) + diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py new file mode 100644 index 000000000..c151831a3 --- /dev/null +++ b/module/threads/DownloadThread.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" + +from Queue import Queue +from time import sleep, time +from traceback import print_exc +from sys import exc_clear +from pycurl import error + +from module.plugins.Base import Fail, Retry +from module.plugins.Hoster import Abort, Reconnect, SkipDownload + +from BaseThread import BaseThread + +class DownloadThread(BaseThread): + """thread for downloading files from 'real' hoster plugins""" + + def __init__(self, manager): + """Constructor""" + BaseThread.__init__(self, manager) + + self.queue = Queue() # job queue + self.active = False + + self.start() + + def run(self): + """run method""" + pyfile = None + + while True: + del pyfile + self.active = self.queue.get() + pyfile = self.active + + if self.active == "quit": + self.active = False + self.m.threads.remove(self) + return True + + try: + if not pyfile.hasPlugin(): continue + #this pyfile was deleted while queueing + + pyfile.plugin.checkForSameFiles(starting=True) + self.log.info(_("Download starts: %s" % pyfile.name)) + + # start download + self.core.hookManager.downloadPreparing(pyfile) + pyfile.plugin.preprocessing(self) + + self.log.info(_("Download finished: %s") % pyfile.name) + self.core.hookManager.downloadFinished(pyfile) + self.core.files.checkPackageFinished(pyfile) + + except NotImplementedError: + self.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + pyfile.setStatus("failed") + pyfile.error = "Plugin does not work" + self.clean(pyfile) + continue + + except Abort: + try: + self.log.info(_("Download aborted: %s") % pyfile.name) + except: + pass + + pyfile.setStatus("aborted") + + self.clean(pyfile) + continue + + except Reconnect: + self.queue.put(pyfile) + #pyfile.req.clearCookies() + + while self.m.reconnecting.isSet(): + sleep(0.5) + + continue + + except Retry, e: + reason = e.args[0] + self.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) + self.queue.put(pyfile) + continue + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + pyfile.setStatus("offline") + self.log.warning(_("Download is offline: %s") % pyfile.name) + elif msg == "temp. offline": + pyfile.setStatus("temp. offline") + self.log.warning(_("Download is temporary offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + + self.core.hookManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + except error, e: + if len(e.args) == 2: + code, msg = e.args + else: + code = 0 + msg = e.args + + self.log.debug("pycurl exception %s: %s" % (code, msg)) + + if code in (7, 18, 28, 52, 56): + self.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) + wait = time() + 60 + + pyfile.waitUntil = wait + pyfile.setStatus("waiting") + while time() < wait: + sleep(1) + if pyfile.abort: + break + + if pyfile.abort: + self.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + self.clean(pyfile) + else: + self.queue.put(pyfile) + + continue + + else: + pyfile.setStatus("failed") + self.log.error("pycurl error %s: %s" % (code, msg)) + if self.core.debug: + print_exc() + self.writeDebugReport(pyfile.plugin.__name__, pyfile) + + self.core.hookManager.downloadFailed(pyfile) + + self.clean(pyfile) + continue + + except SkipDownload, e: + pyfile.setStatus("skipped") + + self.log.info(_("Download skipped: %(name)s due to %(plugin)s") + % {"name": pyfile.name, "plugin": e.message}) + + self.clean(pyfile) + + self.core.files.checkPackageFinished(pyfile) + + self.active = False + self.core.files.save() + + continue + + + except Exception, e: + pyfile.setStatus("failed") + self.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) + + if self.core.debug: + print_exc() + self.writeDebugReport(pyfile.plugin.__name__, pyfile) + + self.core.hookManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + finally: + self.core.files.save() + pyfile.checkIfProcessed() + exc_clear() + + + #pyfile.plugin.req.clean() + + self.active = False + pyfile.finishIfDone() + self.core.files.save() + + + def put(self, job): + """assing job to thread""" + self.queue.put(job) + + + def stop(self): + """stops the thread""" + self.put("quit")
\ No newline at end of file diff --git a/module/threads/HookThread.py b/module/threads/HookThread.py new file mode 100644 index 000000000..bffa72ca0 --- /dev/null +++ b/module/threads/HookThread.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from copy import copy +from traceback import print_exc + +from BaseThread import BaseThread + +class HookThread(BaseThread): + """thread for hooks""" + + def __init__(self, m, function, args, kwargs): + """Constructor""" + BaseThread.__init__(self, m) + + self.f = function + self.args = args + self.kwargs = kwargs + + self.active = [] + + m.localThreads.append(self) + + self.start() + + def getActiveFiles(self): + return self.active + + def addActive(self, pyfile): + """ Adds a pyfile to active list and thus will be displayed on overview""" + if pyfile not in self.active: + self.active.append(pyfile) + + def finishFile(self, pyfile): + if pyfile in self.active: + self.active.remove(pyfile) + + pyfile.finishIfDone() + + def run(self): + try: + try: + self.kwargs["thread"] = self + self.f(*self.args, **self.kwargs) + except TypeError, e: + #dirty method to filter out exceptions + if "unexpected keyword argument 'thread'" not in e.args[0]: + raise + + del self.kwargs["thread"] + self.f(*self.args, **self.kwargs) + except Exception, e: + if hasattr(self.f, "im_self"): + hook = self.f.im_self + hook.logError(_("An Error occured"), e) + if self.m.core.debug: + print_exc() + self.writeDebugReport(hook.__name__, plugin=hook) + + finally: + local = copy(self.active) + for x in local: + self.finishFile(x) + + self.m.localThreads.remove(self)
\ No newline at end of file diff --git a/module/threads/InfoThread.py b/module/threads/InfoThread.py new file mode 100644 index 000000000..7db85803a --- /dev/null +++ b/module/threads/InfoThread.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from time import time +from traceback import print_exc + +from module.Api import OnlineStatus +from module.common.packagetools import parseNames +from module.utils import has_method, accumulate + +from BaseThread import BaseThread + +class InfoThread(BaseThread): + def __init__(self, manager, data, pid=-1, rid=-1): + """Constructor""" + BaseThread.__init__(self, manager) + + self.data = data + self.pid = pid # package id + # [ .. (name, plugin) .. ] + + self.rid = rid #result id + + self.cache = [] #accumulated data + + self.start() + + def run(self): + """run method""" + + plugins = accumulate(self.data) + crypter = {} + + # filter out crypter plugins + for name in self.m.core.pluginManager.getPlugins("crypter"): + if name in plugins: + crypter[name] = plugins[name] + del plugins[name] + + #directly write to database + if self.pid > -1: + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPluginModule(pluginname) + klass = self.m.core.pluginManager.getPluginClass(pluginname) + if has_method(klass, "getInfo"): + self.fetchForPlugin(pluginname, klass, urls, self.updateDB) + self.m.core.files.save() + elif has_method(plugin, "getInfo"): + self.log.debug("Deprecated .getInfo() method on module level, use classmethod instead") + self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) + self.m.core.files.save() + + else: #post the results + for name, urls in crypter: + #attach container content + try: + data = self.decrypt(name, urls) + except: + print_exc() + self.m.log.error("Could not decrypt container.") + data = [] + + accumulate(data, plugins) + + self.m.infoResults[self.rid] = {} + + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + klass = getattr(plugin, pluginname) + if has_method(klass, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) + #force to process cache + if self.cache: + self.updateResult(pluginname, [], True) + elif has_method(plugin, "getInfo"): + self.log.debug("Deprecated .getInfo() method on module level, use staticmethod instead") + self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) + #force to process cache + if self.cache: + self.updateResult(pluginname, [], True) + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateResult(pluginname, result, True) + + self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} + + self.m.timestamp = time() + 5 * 60 + + + def updateDB(self, plugin, result): + self.m.core.files.updateFileInfo(result, self.pid) + + def updateResult(self, plugin, result, force=False): + #parse package name and generate result + #accumulate results + + self.cache.extend(result) + + if len(self.cache) >= 20 or force: + #used for package generating + tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) + for name, size, status, url in self.cache] + + data = parseNames(tmp) + result = {} + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + self.m.setInfoResults(self.rid, result) + + self.cache = [] + + def updateCache(self, plugin, result): + self.cache.extend(result) + + def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): + try: + result = [] #result loaded from cache + process = [] #urls to process + for url in urls: + if url in self.m.infoCache: + result.append(self.m.infoCache[url]) + else: + process.append(url) + + if result: + self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) + cb(pluginname, result) + + if process: + self.m.log.debug("Run Info Fetching for %s" % pluginname) + for result in plugin.getInfo(process): + #result = [ .. (name, size, status, url) .. ] + if not type(result) == list: result = [result] + + for res in result: + self.m.infoCache[res[3]] = res + + cb(pluginname, result) + + self.m.log.debug("Finished Info Fetching for %s" % pluginname) + except Exception, e: + self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % + {"name": pluginname, "err": str(e)}) + if self.m.core.debug: + print_exc() + + # generate default results + if err: + result = [(url, 0, 3, url) for url in urls] + cb(pluginname, result) + + + def decrypt(self, plugin, urls): + self.m.log.debug("Pre decrypting %s" % plugin) + klass = self.m.core.pluginManager.loadClass("crypter", plugin) + + # only decrypt files + if has_method(klass, "decryptFile"): + urls = p.decrypt(urls) + data, crypter = self.m.core.pluginManager.parseUrls(urls) + return data + + return [] diff --git a/module/ThreadManager.py b/module/threads/ThreadManager.py index 8937f4a29..f8b5c0aba 100644 --- a/module/ThreadManager.py +++ b/module/threads/ThreadManager.py @@ -28,11 +28,14 @@ from random import choice import pycurl -import PluginThread from module.PyFile import PyFile from module.network.RequestFactory import getURL -from module.utils import freeSpace, lock +from module.utils import lock, uniqify +from module.utils.fs import free_space +from DecrypterThread import DecrypterThread +from DownloadThread import DownloadThread +from InfoThread import InfoThread class ThreadManager: """manages the download threads, assign jobs, reconnect etc""" @@ -63,42 +66,43 @@ class ThreadManager: # threads which are fetching hoster results self.infoResults = {} - #timeout for cache purge + # timeout for cache purge self.timestamp = 0 pycurl.global_init(pycurl.GLOBAL_DEFAULT) - for i in range(0, self.core.config.get("download", "max_downloads")): + for i in range(self.core.config.get("download", "max_downloads")): self.createThread() def createThread(self): """create a download thread""" - thread = PluginThread.DownloadThread(self) + thread = DownloadThread(self) self.threads.append(thread) def createInfoThread(self, data, pid): - """ - start a thread whichs fetches online status and other infos - data = [ .. () .. ] - """ + """ start a thread whichs fetches online status and other infos """ self.timestamp = time() + 5 * 60 - - PluginThread.InfoThread(self, data, pid) + if data: InfoThread(self, data, pid) @lock - def createResultThread(self, data, add=False): + def createResultThread(self, data): """ creates a thread to fetch online status, returns result id """ self.timestamp = time() + 5 * 60 rid = self.resultIDs self.resultIDs += 1 - PluginThread.InfoThread(self, data, rid=rid, add=add) + InfoThread(self, data, rid=rid) return rid + @lock + def createDecryptThread(self, data, pid): + """ Start decrypting of entered data, all links in one package are accumulated to one thread.""" + if data: DecrypterThread(self, data, pid) + @lock def getInfoResult(self, rid): @@ -156,7 +160,6 @@ class ThreadManager: self.infoResults.clear() self.log.debug("Cleared Result cache") - #---------------------------------------------------------------------- def tryReconnect(self): """checks if reconnect needed""" @@ -227,7 +230,6 @@ class ThreadManager: return ip - #---------------------------------------------------------------------- def checkThreadCount(self): """checks if there are need for increasing or reducing thread count""" @@ -251,7 +253,7 @@ class ThreadManager: self.log.debug("Cleaned up pycurl") return True - #---------------------------------------------------------------------- + def assignJob(self): """assing a job to a thread if possible""" @@ -262,14 +264,10 @@ class ThreadManager: free = [x for x in self.threads if not x.active] - inuse = set([(x.active.pluginname,self.getLimit(x)) for x in self.threads if x.active and x.active.hasPlugin() and x.active.plugin.account]) - inuse = map(lambda x : (x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) ,inuse) - onlimit = [x[0] for x in inuse if x[1] > 0 and x[2] >= x[1]] - - occ = [x.active.pluginname for x in self.threads if x.active and x.active.hasPlugin() and not x.active.plugin.multiDL] + onlimit + inuse = [(x.active.pluginname, x.active.plugin.getDownloadLimit()) for x in self.threads if x.active and x.active.hasPlugin()] + inuse = [(x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])) for x in inuse] + occ = tuple(sorted(uniqify([x[0] for x in inuse if 0 < x[1] <= x[2]]))) - occ.sort() - occ = tuple(set(occ)) job = self.core.files.getJob(occ) if job: try: @@ -282,36 +280,20 @@ class ThreadManager: job.release() return - if job.plugin.__type__ == "hoster": - spaceLeft = freeSpace(self.core.config["general"]["download_folder"]) / 1024 / 1024 - if spaceLeft < self.core.config["general"]["min_free_space"]: - self.log.warning(_("Not enough space left on device")) - self.pause = True - - if free and not self.pause: - thread = free[0] - #self.downloaded += 1 - - thread.put(job) - else: - #put job back - if occ not in self.core.files.jobCache: - self.core.files.jobCache[occ] = [] - self.core.files.jobCache[occ].append(job.id) - - #check for decrypt jobs - job = self.core.files.getDecryptJob() - if job: - job.initPlugin() - thread = PluginThread.DecrypterThread(self, job) - + spaceLeft = free_space(self.core.config["general"]["download_folder"]) / 1024 / 1024 + if spaceLeft < self.core.config["general"]["min_free_space"]: + self.log.warning(_("Not enough space left on device")) + self.pause = True + if free and not self.pause: + thread = free[0] + #self.downloaded += 1 + thread.put(job) else: - thread = PluginThread.DecrypterThread(self, job) - - def getLimit(self, thread): - limit = thread.active.plugin.account.getAccountData(thread.active.plugin.user)["options"].get("limitDL",["0"])[0] - return int(limit) + #put job back + if occ not in self.core.files.jobCache: + self.core.files.jobCache[occ] = [] + self.core.files.jobCache[occ].append(job.id) def cleanup(self): """do global cleanup, should be called when finished with pycurl""" diff --git a/module/threads/__init__.py b/module/threads/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/module/threads/__init__.py diff --git a/module/unescape.py b/module/unescape.py deleted file mode 100644 index d8999e077..000000000 --- a/module/unescape.py +++ /dev/null @@ -1,3 +0,0 @@ -from module.utils import html_unescape -#deprecated -unescape = html_unescape
\ No newline at end of file diff --git a/module/Utils.py b/module/utils/__init__.py index 8748b7693..592bdbd7e 100644 --- a/module/Utils.py +++ b/module/utils/__init__.py @@ -3,27 +3,31 @@ """ Store all usefull functions here """ import os -import sys import time import re -from os.path import join from string import maketrans +from itertools import islice from htmlentitydefs import name2codepoint -def chmod(*args): - try: - os.chmod(*args) - except: - pass - - def decode(string): """ decode string with utf if possible """ try: - return string.decode("utf8", "replace") + if type(string) == str: + return string.decode("utf8", "replace") + else: + return string except: return string +def encode(string): + """ decode string to utf if possible """ + try: + if type(string) == unicode: + return string.encode("utf8", "replace") + else: + return string + except: + return string def remove_chars(string, repl): """ removes all chars in repl from string""" @@ -33,34 +37,6 @@ def remove_chars(string, repl): return string.translate(dict([(ord(s), None) for s in repl])) -def save_path(name): - #remove some chars - if os.name == 'nt': - return remove_chars(name, '/\\?%*:|"<>') - else: - return remove_chars(name, '/\\"') - - -def save_join(*args): - """ joins a path, encoding aware """ - return fs_encode(join(*[x if type(x) == unicode else decode(x) for x in args])) - - -# File System Encoding functions: -# Use fs_encode before accesing files on disk, it will encode the string properly - -if sys.getfilesystemencoding().startswith('ANSI'): - def fs_encode(string): - try: - string = string.encode('utf-8') - finally: - return string - - fs_decode = decode #decode utf8 - -else: - fs_encode = fs_decode = lambda x: x # do nothing - def get_console_encoding(enc): if os.name == "nt": if enc == "cp65001": # aka UTF-8 @@ -83,6 +59,8 @@ def compare_time(start, end): elif start < now > end < start: return True else: return False +def to_list(value): + return value if type(value) == list else [value] def formatSize(size): """formats size of bytes""" @@ -98,36 +76,10 @@ def formatSize(size): def formatSpeed(speed): return formatSize(speed) + "/s" - -def freeSpace(folder): - if os.name == "nt": - import ctypes - - free_bytes = ctypes.c_ulonglong(0) - ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(folder), None, None, ctypes.pointer(free_bytes)) - return free_bytes.value - else: - from os import statvfs - - s = statvfs(folder) - return s.f_bsize * s.f_bavail - - -def uniqify(seq, idfun=None): -# order preserving - if idfun is None: - def idfun(x): return x - seen = {} - result = [] - for item in seq: - marker = idfun(item) - # in old Python versions: - # if seen.has_key(marker) - # but in new ones: - if marker in seen: continue - seen[marker] = 1 - result.append(item) - return result +def uniqify(seq): #by Dave Kirby + """ removes duplicates from list, preserve order """ + seen = set() + return [x for x in seq if x not in seen and not seen.add(x)] def parseFileSize(string, unit=None): #returns bytes @@ -168,6 +120,13 @@ def lock(func): return new +def chunks(iterable, size): + it = iter(iterable) + item = list(islice(it, size)) + while item: + yield item + item = list(islice(it, size)) + def fixup(m): text = m.group(0) @@ -191,11 +150,57 @@ def fixup(m): return text # leave as is +def has_method(obj, name): + """ checks if 'name' was defined in obj, (false if it was inhereted) """ + return name in obj.__dict__ + +def accumulate(it, inv_map=None): + """ accumulate (key, value) data to {value : [keylist]} dictionary """ + if not inv_map: + inv_map = {} + + for key, value in it: + if value in inv_map: + inv_map[value].append(key) + else: + inv_map[value] = [key] + + return inv_map + +def to_string(value): + return str(value) if not isinstance(value, basestring) else value + +def to_int(string): + """ return int from string or 0 """ + try: + return int(string) + except ValueError: + return 0 + +def from_string(value, typ=None): + """ cast value to given type, unicode for strings """ + + # value is no string + if not isinstance(value, basestring): + return value + + value = decode(value) + + if typ == "int": + return int(value) + elif typ == "bool": + return True if value.lower() in ("1", "true", "on", "an", "yes") else False + elif typ == "time": + if not value: value = "0:00" + if not ":" in value: value += ":00" + return value + else: + return value + + def html_unescape(text): """Removes HTML or XML character references and entities from a text string""" return re.sub("&#?\w+;", fixup, text) if __name__ == "__main__": - print freeSpace(".") - print remove_chars("ab'cdgdsf''ds'", "'ghd") diff --git a/module/utils/fs.py b/module/utils/fs.py new file mode 100644 index 000000000..350283275 --- /dev/null +++ b/module/utils/fs.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +import os +import sys +from os.path import join +from . import decode, remove_chars + +# File System Encoding functions: +# Use fs_encode before accesing files on disk, it will encode the string properly + +if sys.getfilesystemencoding().startswith('ANSI'): + def fs_encode(string): + if type(string) == unicode: + return string.encode('utf8') + return string + + fs_decode = decode #decode utf8 + +else: + fs_encode = fs_decode = lambda x: x # do nothing + +# FS utilities +def chmod(path, mode): + try: + return os.chmod(fs_encode(path), mode) + except : + pass + +def chown(path, uid, gid): + return os.chown(fs_encode(path), uid, gid) + +def remove(path): + return os.remove(fs_encode(path)) + +def exists(path): + return os.path.exists(fs_encode(path)) + +def makedirs(path, mode=0755): + return os.makedirs(fs_encode(path), mode) + +def listdir(path): + return os.listdir(fs_encode(path)) + +def save_filename(name): + #remove some chars + if os.name == 'nt': + return remove_chars(name, '/\\?%*:|"<>') + else: + return remove_chars(name, '/\\"') + +def stat(name): + return os.stat(fs_encode(name)) + +def save_join(*args): + """ joins a path, encoding aware """ + return fs_encode(join(*[x if type(x) == unicode else decode(x) for x in args])) + +def free_space(folder): + folder = fs_encode(folder) + + if os.name == "nt": + import ctypes + + free_bytes = ctypes.c_ulonglong(0) + ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(folder), None, None, ctypes.pointer(free_bytes)) + return free_bytes.value + else: + from os import statvfs + + s = statvfs(folder) + return s.f_bsize * s.f_bavail diff --git a/module/web/ServerThread.py b/module/web/ServerThread.py index 84667e5f6..8b59ca01b 100644 --- a/module/web/ServerThread.py +++ b/module/web/ServerThread.py @@ -93,6 +93,13 @@ class WebServer(threading.Thread): webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key) def start_fcgi(self): + + from flup.server.threadedserver import ThreadedServer + + def noop(*args, **kwargs): + pass + + ThreadedServer._installSignalHandlers = noop self.core.log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) webinterface.run_fcgi(host=self.host, port=self.port) diff --git a/module/web/api_app.py b/module/web/api_app.py index 1629c1677..affcdb39a 100644 --- a/module/web/api_app.py +++ b/module/web/api_app.py @@ -11,6 +11,7 @@ from utils import toDict, set_session from webinterface import PYLOAD from module.common.json_layer import json +from module.utils import remove_chars from module.lib.SafeEval import const_eval as literal_eval from module.Api import BaseObject @@ -24,16 +25,17 @@ class TBaseEncoder(json.JSONEncoder): # accepting positional arguments, as well as kwargs via post and get - -@route("/api/:func:args#[a-zA-Z0-9\-_/\"'\[\]%{}]*#") -@route("/api/:func:args#[a-zA-Z0-9\-_/\"'\[\]%{}]*#", method="POST") +# only forbidden path symbol are "?", which is used to seperate GET data and # +@route("/api/<func><args:re:[^#?]*>") +@route("/api/<func><args:re:[^#?]*>", method="POST") def call_api(func, args=""): response.headers.replace("Content-type", "application/json") response.headers.append("Cache-Control", "no-cache, must-revalidate") s = request.environ.get('beaker.session') if 'session' in request.POST: - s = s.get_by_id(request.POST['session']) + # removes "' so it works on json strings + s = s.get_by_id(remove_chars(request.POST['session'], "'\"")) if not s or not s.get("authenticated", False): return HTTPError(403, json.dumps("Forbidden")) @@ -63,7 +65,7 @@ def callApi(func, *args, **kwargs): result = getattr(PYLOAD, func)(*[literal_eval(x) for x in args], **dict([(x, literal_eval(y)) for x, y in kwargs.iteritems()])) - # null is invalid json response + # null is invalid json response if result is None: result = True return json.dumps(result, cls=TBaseEncoder) diff --git a/module/web/cnl_app.py b/module/web/cnl_app.py index d8f7c1180..fe308ec04 100644 --- a/module/web/cnl_app.py +++ b/module/web/cnl_app.py @@ -18,7 +18,7 @@ except: def local_check(function): def _view(*args, **kwargs): if request.environ.get('REMOTE_ADDR', "0") in ('127.0.0.1', 'localhost') \ - or request.environ.get('HTTP_HOST','0') == '127.0.0.1:9666': + or request.environ.get('HTTP_HOST','0') in ('127.0.0.1:9666', 'localhost:9666'): return function(*args, **kwargs) else: return HTTPError(403, "Forbidden") diff --git a/module/web/json_app.py b/module/web/json_app.py index f3626405c..5acafe153 100644 --- a/module/web/json_app.py +++ b/module/web/json_app.py @@ -179,11 +179,7 @@ def add_package(): links = map(lambda x: x.strip(), links) links = filter(lambda x: x != "", links) - pack = PYLOAD.addPackage(name, links, queue) - if pw: - pw = pw.decode("utf8", "ignore") - data = {"password": pw} - PYLOAD.setPackageData(pack, data) + PYLOAD.addPackage(name, links, queue, pw.decode("utf8", "ignore")) @route("/json/move_package/<dest:int>/<id:int>") @@ -232,38 +228,23 @@ def set_captcha(): return {'captcha': False} -@route("/json/load_config/:category/:section") +@route("/json/load_config/:section") @login_required("SETTINGS") -def load_config(category, section): - conf = None - if category == "general": - conf = PYLOAD.getConfigDict() - elif category == "plugin": - conf = PYLOAD.getPluginConfigDict() +def load_config(section): + data = PYLOAD.configureSection(section) + return render_to_response("settings_item.html", {"section": data}) - for key, option in conf[section].iteritems(): - if key in ("desc","outline"): continue - if ";" in option["type"]: - option["list"] = option["type"].split(";") - - option["value"] = decode(option["value"]) - - return render_to_response("settings_item.html", {"skey": section, "section": conf[section]}) - - -@route("/json/save_config/:category", method="POST") +@route("/json/save_config", method="POST") @login_required("SETTINGS") -def save_config(category): +def save_config(): for key, value in request.POST.iteritems(): try: section, option = key.split("|") except: continue - if category == "general": category = "core" - - PYLOAD.setConfigValue(section, option, decode(value), category) + PYLOAD.setConfigValue(section, option, decode(value)) @route("/json/add_account", method="POST") @@ -293,9 +274,9 @@ def update_accounts(): if action == "password": PYLOAD.updateAccount(plugin, user, value) elif action == "time" and "-" in value: - PYLOAD.updateAccount(plugin, user, options={"time": [value]}) + PYLOAD.updateAccount(plugin, user, options={"time": value}) elif action == "limitdl" and value.isdigit(): - PYLOAD.updateAccount(plugin, user, options={"limitDL": [value]}) + PYLOAD.updateAccount(plugin, user, options={"limitDL": value}) elif action == "delete": deleted.append((plugin,user)) PYLOAD.removeAccount(plugin, user) diff --git a/module/web/media/js/settings.coffee b/module/web/media/js/settings.coffee index 9205233e3..04d352dae 100644 --- a/module/web/media/js/settings.coffee +++ b/module/web/media/js/settings.coffee @@ -51,7 +51,7 @@ class SettingsUI new Request({ "method" : "get" - "url" : "/json/load_config/#{category}/#{section}" + "url" : "/json/load_config/#{section}" "onSuccess": (data) => target.set "html", data target.reveal() @@ -65,7 +65,7 @@ class SettingsUI form.set "send", { "method": "post" - "url": "/json/save_config/#{category}" + "url": "/json/save_config" "onSuccess" : -> root.notify.alert '{{ _("Settings saved.")}}', { 'className': 'success' diff --git a/module/web/media/js/settings.js b/module/web/media/js/settings.js index 9191fac72..3604c38b0 100644 --- a/module/web/media/js/settings.js +++ b/module/web/media/js/settings.js @@ -1,3 +1,3 @@ {% autoescape true %} -var SettingsUI,root;var __bind=function(a,b){return function(){return a.apply(b,arguments)}};root=this;window.addEvent("domready",function(){root.accountDialog=new MooDialog({destroyOnHide:false});root.accountDialog.setContent($("account_box"));new TinyTab($$("#toptabs li a"),$$("#tabs-body > span"));$$("ul.nav").each(function(a){return new MooDropMenu(a,{onOpen:function(b){return b.fade("in")},onClose:function(b){return b.fade("out")},onInitialize:function(b){return b.fade("hide").set("tween",{duration:500})}})});return new SettingsUI()});SettingsUI=(function(){function a(){var c,e,b,d;this.menu=$$("#general-menu li");this.menu.append($$("#plugin-menu li"));this.name=$("tabsback");this.general=$("general_form_content");this.plugin=$("plugin_form_content");d=this.menu;for(e=0,b=d.length;e<b;e++){c=d[e];c.addEvent("click",this.menuClick.bind(this))}$("general|submit").addEvent("click",this.configSubmit.bind(this));$("plugin|submit").addEvent("click",this.configSubmit.bind(this));$("account_add").addEvent("click",function(f){root.accountDialog.open();return f.stop()});$("account_reset").addEvent("click",function(f){return root.accountDialog.close()});$("account_add_button").addEvent("click",this.addAccount.bind(this));$("account_submit").addEvent("click",this.submitAccounts.bind(this))}a.prototype.menuClick=function(h){var c,b,g,f,d;d=h.target.get("id").split("|"),c=d[0],g=d[1];b=h.target.get("text");f=c==="general"?this.general:this.plugin;f.dissolve();return new Request({method:"get",url:"/json/load_config/"+c+"/"+g,onSuccess:__bind(function(e){f.set("html",e);f.reveal();return this.name.set("text",b)},this)}).send()};a.prototype.configSubmit=function(d){var c,b;c=d.target.get("id").split("|")[0];b=$(""+c+"_form");b.set("send",{method:"post",url:"/json/save_config/"+c,onSuccess:function(){return root.notify.alert('{{ _("Settings saved.")}}',{className:"success"})},onFailure:function(){return root.notify.alert('{{ _("Error occured.")}}',{className:"error"})}});b.send();return d.stop()};a.prototype.addAccount=function(c){var b;b=$("add_account_form");b.set("send",{method:"post",onSuccess:function(){return window.location.reload()},onFailure:function(){return root.notify.alert('{{_("Error occured.")}}',{className:"error"})}});b.send();return c.stop()};a.prototype.submitAccounts=function(c){var b;b=$("account_form");b.set("send",{method:"post",onSuccess:function(){return window.location.reload()},onFailure:function(){return root.notify.alert('{{ _("Error occured.") }}',{className:"error"})}});b.send();return c.stop()};return a})(); +var SettingsUI,root;var __bind=function(a,b){return function(){return a.apply(b,arguments)}};root=this;window.addEvent("domready",function(){root.accountDialog=new MooDialog({destroyOnHide:false});root.accountDialog.setContent($("account_box"));new TinyTab($$("#toptabs li a"),$$("#tabs-body > span"));$$("ul.nav").each(function(a){return new MooDropMenu(a,{onOpen:function(b){return b.fade("in")},onClose:function(b){return b.fade("out")},onInitialize:function(b){return b.fade("hide").set("tween",{duration:500})}})});return new SettingsUI()});SettingsUI=(function(){function a(){var c,e,b,d;this.menu=$$("#general-menu li");this.menu.append($$("#plugin-menu li"));this.name=$("tabsback");this.general=$("general_form_content");this.plugin=$("plugin_form_content");d=this.menu;for(e=0,b=d.length;e<b;e++){c=d[e];c.addEvent("click",this.menuClick.bind(this))}$("general|submit").addEvent("click",this.configSubmit.bind(this));$("plugin|submit").addEvent("click",this.configSubmit.bind(this));$("account_add").addEvent("click",function(f){root.accountDialog.open();return f.stop()});$("account_reset").addEvent("click",function(f){return root.accountDialog.close()});$("account_add_button").addEvent("click",this.addAccount.bind(this));$("account_submit").addEvent("click",this.submitAccounts.bind(this))}a.prototype.menuClick=function(h){var c,b,g,f,d;d=h.target.get("id").split("|"),c=d[0],g=d[1];b=h.target.get("text");f=c==="general"?this.general:this.plugin;f.dissolve();return new Request({method:"get",url:"/json/load_config/"+g,onSuccess:__bind(function(e){f.set("html",e);f.reveal();return this.name.set("text",b)},this)}).send()};a.prototype.configSubmit=function(d){var c,b;c=d.target.get("id").split("|")[0];b=$(""+c+"_form");b.set("send",{method:"post",url:"/json/save_config",onSuccess:function(){return root.notify.alert('{{ _("Settings saved.")}}',{className:"success"})},onFailure:function(){return root.notify.alert('{{ _("Error occured.")}}',{className:"error"})}});b.send();return d.stop()};a.prototype.addAccount=function(c){var b;b=$("add_account_form");b.set("send",{method:"post",onSuccess:function(){return window.location.reload()},onFailure:function(){return root.notify.alert('{{_("Error occured.")}}',{className:"error"})}});b.send();return c.stop()};a.prototype.submitAccounts=function(c){var b;b=$("account_form");b.set("send",{method:"post",onSuccess:function(){return window.location.reload()},onFailure:function(){return root.notify.alert('{{ _("Error occured.") }}',{className:"error"})}});b.send();return c.stop()};return a})(); {% endautoescape %}
\ No newline at end of file diff --git a/module/web/middlewares.py b/module/web/middlewares.py index e0e6c3102..57023dbdb 100644 --- a/module/web/middlewares.py +++ b/module/web/middlewares.py @@ -90,14 +90,14 @@ class GzipResponse(object): cl = int(cl) else: cl = 201 - self.compressible = False - if ct and (ct.startswith('text/') or ct.startswith('application/')) \ - and 'zip' not in ct and cl > 200: - self.compressible = True + if ce: self.compressible = False - if self.compressible: + elif ct and (ct.startswith('text/') or ct.startswith('application/')) \ + and 'zip' not in ct and 200 < cl < 1024*1024: + self.compressible = True headers.append(('content-encoding', 'gzip')) + remove_header(headers, 'content-length') self.headers = headers self.status = status diff --git a/module/web/pyload_app.py b/module/web/pyload_app.py index df4a4b3d4..dcfc3266e 100644 --- a/module/web/pyload_app.py +++ b/module/web/pyload_app.py @@ -22,7 +22,6 @@ from operator import itemgetter, attrgetter import time import os import sys -from os import listdir from os.path import isdir, isfile, join, abspath from sys import getfilesystemencoding from urllib import unquote @@ -36,7 +35,8 @@ from utils import render_to_response, parse_permissions, parse_userdata, \ from filters import relpath, unquotepath -from module.utils import formatSize, save_join, fs_encode, fs_decode +from module.utils import formatSize +from module.utils.fs import save_join, fs_encode, fs_decode, listdir, free_space # Helper @@ -189,7 +189,7 @@ def collector(): def downloads(): root = PYLOAD.getConfigValue("general", "download_folder") - if not isdir(root): + if not isdir(fs_encode(root)): return base([_('Download directory not found.')]) data = { 'folder': [], @@ -241,45 +241,40 @@ def get_download(path): @route("/settings") @login_required('SETTINGS') def config(): - conf = PYLOAD.getConfig() - plugin = PYLOAD.getPluginConfig() + conf = PYLOAD.getConfigPointer() conf_menu = [] plugin_menu = [] - for entry in sorted(conf.keys()): - conf_menu.append((entry, conf[entry].description)) + for section, data in sorted(conf.getBaseSections()): + conf_menu.append((section, data.name)) - for entry in sorted(plugin.keys()): - plugin_menu.append((entry, plugin[entry].description)) + for section, data in sorted(conf.getPluginSections()): + plugin_menu.append((section, data.name)) accs = PYLOAD.getAccounts(False) + # prefix attributes with _, because we would change them directly on the object otherweise for data in accs: if data.trafficleft == -1: - data.trafficleft = _("unlimited") + data._trafficleft = _("unlimited") elif not data.trafficleft: - data.trafficleft = _("not available") + data._trafficleft = _("not available") else: - data.trafficleft = formatSize(data.trafficleft * 1024) + data._trafficleft = formatSize(data.trafficleft * 1024) if data.validuntil == -1: - data.validuntil = _("unlimited") - elif not data.validuntil : - data.validuntil = _("not available") + data._validuntil = _("unlimited") + elif not data.validuntil: + data._validuntil = _("not available") else: t = time.localtime(data.validuntil) - data.validuntil = time.strftime("%d.%m.%Y", t) + data._validuntil = time.strftime("%d.%m.%Y", t) - if "time" in data.options: - try: - data.options["time"] = data.options["time"][0] - except: - data.options["time"] = "0:00-0:00" + if not data.options["time"]: + data.options["time"] = "0:00-0:00" - if "limitDL" in data.options: - data.options["limitdl"] = data.options["limitDL"][0] - else: + if not data.options["limitDL"]: data.options["limitdl"] = "0" return render_to_response('settings.html', @@ -511,9 +506,10 @@ def setup(): return render_to_response('setup.html', {"user": False, "perms": False}) +@login_required("STATUS") @route("/info") def info(): - conf = PYLOAD.getConfigDict() + conf = PYLOAD.getConfigPointer() if hasattr(os, "uname"): extra = os.uname() @@ -524,10 +520,10 @@ def info(): "os": " ".join((os.name, sys.platform) + extra), "version": PYLOAD.getServerVersion(), "folder": abspath(PYLOAD_DIR), "config": abspath(""), - "download": abspath(conf["general"]["download_folder"]["value"]), + "download": abspath(conf["general"]["download_folder"]), "freespace": formatSize(PYLOAD.freeSpace()), - "remote": conf["remote"]["port"]["value"], - "webif": conf["webinterface"]["port"]["value"], - "language": conf["general"]["language"]["value"]} + "remote": conf["remote"]["port"], + "webif": conf["webinterface"]["port"], + "language": conf["general"]["language"]} return render_to_response("info.html", data, [pre_processor]) diff --git a/module/web/templates/default/base.html b/module/web/templates/default/base.html index 0b20ecdb0..1f77c04ba 100644 --- a/module/web/templates/default/base.html +++ b/module/web/templates/default/base.html @@ -162,7 +162,7 @@ <hr style="clear: both;" />
-<div id="foot">© 2008-2011 pyLoad Team
+<div id="foot">© 2008-2012 pyLoad Team
<a href="#top" class="action top" accesskey="x"><span>{{_("Back to top")}}</span></a><br />
<!--<div class="breadcrumbs"></div>-->
diff --git a/module/web/templates/default/queue.html b/module/web/templates/default/queue.html index c88fa3568..9403a8019 100644 --- a/module/web/templates/default/queue.html +++ b/module/web/templates/default/queue.html @@ -50,7 +50,11 @@ document.addEvent("domready", function(){ <img title="{{_("Move Package")}}" style="margin-left: -10px; cursor: pointer" height="12px" src="/media/default/img/package_go.png" />
</span>
</div>
- {% set progress = (package.linksdone * 100) / package.linkstotal %}
+ {% if package.linkstotal %}
+ {% set progress = (package.linksdone * 100) / package.linkstotal %}
+ {% else %}
+ {% set progress = 0 %}
+ {% endif %}
<div id="progress" style="border-radius: 4px; border: 1px solid #AAAAAA; width: 50%; height: 1em">
<div style="width: {{ progress }}%; height: 100%; background-color: #add8e6;"></div>
diff --git a/module/web/templates/default/settings.html b/module/web/templates/default/settings.html index a4443025a..be320970b 100644 --- a/module/web/templates/default/settings.html +++ b/module/web/templates/default/settings.html @@ -102,18 +102,18 @@ {% for account in conf.accs %} - {% set plugin = account.type %} + {% set plugin = account.__name__ %} <tr> <td> <span style="padding:5px">{{ plugin }}</span> </td> - <td><label for="{{plugin}}|password;{{account.login}}" - style="color:#424242;">{{ account.login }}</label></td> + <td><label for="{{plugin}}|password;{{account.loginname}}" + style="color:#424242;">{{ account.loginname }}</label></td> <td> - <input id="{{plugin}}|password;{{account.login}}" - name="{{plugin}}|password;{{account.login}}" - type="password" value="{{account.password}}" size="12"/> + <input id="{{plugin}}|password;{{account.loginname}}" + name="{{plugin}}|password;{{account.loginname}}" + type="password" value="" size="12"/> </td> <td> {% if account.valid %} @@ -137,27 +137,27 @@ </td> <td> <span style="font-weight: bold;"> - {{ account.validuntil }} + {{ account._validuntil }} </span> </td> <td> <span style="font-weight: bold;"> - {{ account.trafficleft }} + {{ account._trafficleft }} </span> </td> <td> - <input id="{{plugin}}|time;{{account.login}}" - name="{{plugin}}|time;{{account.login}}" type="text" - size="7" value="{{account.time}}"/> + <input id="{{plugin}}|time;{{account.loginname}}" + name="{{plugin}}|time;{{account.loginname}}" type="text" + size="7" value="{{account.options.time}}"/> </td> <td> - <input id="{{plugin}}|limitdl;{{account.login}}" - name="{{plugin}}|limitdl;{{account.login}}" type="text" - size="2" value="{{account.limitdl}}"/> + <input id="{{plugin}}|limitdl;{{account.loginname}}" + name="{{plugin}}|limitdl;{{account.loginname}}" type="text" + size="2" value="{{account.options.limitdl}}"/> </td> <td> - <input id="{{plugin}}|delete;{{account.login}}" - name="{{plugin}}|delete;{{account.login}}" type="checkbox" + <input id="{{plugin}}|delete;{{account.loginname}}" + name="{{plugin}}|delete;{{account.loginname}}" type="checkbox" value="True"/> </td> </tr> diff --git a/module/web/templates/default/settings_item.html b/module/web/templates/default/settings_item.html index 813383343..b3d7fe334 100644 --- a/module/web/templates/default/settings_item.html +++ b/module/web/templates/default/settings_item.html @@ -1,12 +1,13 @@ <table class="settable"> - {% if section.outline %} - <tr><th colspan="2">{{ section.outline }}</th></tr> + {% if section.description %} + <tr><th colspan="2">{{ section.description }}</th></tr> {% endif %} - {% for okey, option in section.iteritems() %} - {% if okey not in ("desc","outline") %} + {% for option in section.items %} + {% set okey = option.name %} + {% set skey = section.name %} <tr> - <td><label for="{{skey}}|{{okey}}" - style="color:#424242;">{{ option.desc }}:</label></td> + <td><label for="{{section.name}}|{{option.name}}" + style="color:#424242;">{{ option.long_name }}:</label></td> <td> {% if option.type == "bool" %} <select id="{{skey}}|{{okey}}" name="{{skey}}|{{okey}}"> @@ -17,7 +18,7 @@ </select> {% elif ";" in option.type %} <select id="{{skey}}|{{okey}}" name="{{skey}}|{{okey}}"> - {% for entry in option.list %} + {% for entry in option.type.split(";") %} <option {% if option.value == entry %} selected="selected" {% endif %}>{{ entry }}</option> {% endfor %} @@ -43,6 +44,5 @@ {% endif %} </td> </tr> - {% endif %} {% endfor %} </table>
\ No newline at end of file diff --git a/pavement.py b/pavement.py index ac9a6fa1a..4b5ccb883 100644 --- a/pavement.py +++ b/pavement.py @@ -3,7 +3,25 @@ from paver.easy import * from paver.setuputils import setup -from paver.doctools import cog +try: + from paver.doctools import cog +except: + cog = None + +import fnmatch + +# patch to let it support list of patterns +def new_fnmatch(self, pattern): + if type(pattern) == list: + for p in pattern: + if fnmatch.fnmatch(self.name, p): + return True + return False + else: + return fnmatch.fnmatch(self.name, pattern) + +path.fnmatch = new_fnmatch + import sys import re @@ -23,7 +41,7 @@ if sys.version_info <= (2, 5): setup( name="pyload", - version="0.4.9", + version="0.5.0", description='Fast, lightweight and full featured download manager.', long_description=open(PROJECT_DIR / "README").read(), keywords = ('pyload', 'download-manager', 'one-click-hoster', 'download'), @@ -86,7 +104,7 @@ options( virtual="virtualenv2", ), cog=Bunch( - pattern="*.py", + pattern=["*.py", "*.rst"], ) ) @@ -147,7 +165,7 @@ def get_source(options): @task -@needs('clean', 'generate_setup', 'minilib', 'get_source', 'setuptools.command.sdist') +@needs('clean', 'generate_setup', 'get_source', 'setuptools.command.sdist') def sdist(): """ Build source code package with distutils """ @@ -242,7 +260,8 @@ def generate_locale(): @task def tests(): - call(["nosetests2"]) + """ Run nosetests """ + call(["tests/nosetests.sh"]) @task def virtualenv(options): @@ -263,7 +282,7 @@ def clean_env(): @task -@needs('generate_setup', 'minilib', 'get_source', 'virtualenv') +@needs('generate_setup', 'get_source', 'virtualenv') def env_install(): """Install pyLoad into the virtualenv""" venv = options.virtualenv diff --git a/paver-minilib.zip b/paver-minilib.zip Binary files differnew file mode 100644 index 000000000..15cb0ecb3 --- /dev/null +++ b/paver-minilib.zip diff --git a/pyLoadCli.py b/pyLoadCli.py index 079cee19c..d68b5faec 100755 --- a/pyLoadCli.py +++ b/pyLoadCli.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -#Copyright (C) 2011 RaNaN +#Copyright (C) 2012 RaNaN # #This program is free software; you can redistribute it and/or modify #it under the terms of the GNU General Public License as published by @@ -289,14 +289,14 @@ class Cli: print _("Please use this syntax: add <Package name> <link> <link2> ...") return - self.client.addPackage(args[0], args[1:], Destination.Queue) + self.client.addPackage(args[0], args[1:], Destination.Queue, "") elif command == "add_coll": if len(args) < 2: print _("Please use this syntax: add <Package name> <link> <link2> ...") return - self.client.addPackage(args[0], args[1:], Destination.Collector) + self.client.addPackage(args[0], args[1:], Destination.Collector, "") elif command == "del_file": self.client.deleteFiles([int(x) for x in args]) @@ -392,7 +392,7 @@ class RefreshThread(Thread): def print_help(config): print - print "pyLoadCli Copyright (c) 2008-2011 the pyLoad Team" + print "pyLoadCli Copyright (c) 2008-2012 the pyLoad Team" print print "Usage: [python] pyLoadCli.py [options] [command]" print diff --git a/pyLoadCore.py b/pyLoadCore.py index 35cac4682..e79da3fc3 100755 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -20,31 +20,34 @@ @author: mkaay @version: v0.4.9 """ -CURRENT_VERSION = '0.4.9' +CURRENT_VERSION = '0.4.9.9-dev' import __builtin__ from getopt import getopt, GetoptError -import module.common.pylgettext as gettext from imp import find_module import logging import logging.handlers import os -from os import _exit, execl, getcwd, makedirs, remove, sep, walk, chdir, close -from os.path import exists, join +from os import _exit, execl, getcwd, remove, walk, chdir, close import signal -import subprocess import sys from sys import argv, executable, exit from time import time, sleep from traceback import print_exc +import locale +locale.locale_alias = locale.windows_locale = {} #save ~100kb ram, no known sideeffects for now + +import subprocess +subprocess.__doc__ = None # the module with the largest doc we are using + from module import InitHomeDir from module.plugins.AccountManager import AccountManager -from module.CaptchaManager import CaptchaManager -from module.ConfigParser import ConfigParser +from module.interaction.CaptchaManager import CaptchaManager +from module.config.ConfigParser import ConfigParser from module.plugins.PluginManager import PluginManager -from module.PullEvents import PullManager +from module.interaction.EventManager import EventManager from module.network.RequestFactory import RequestFactory from module.web.ServerThread import WebServer from module.Scheduler import Scheduler @@ -53,11 +56,17 @@ from module import remote from module.remote.RemoteManager import RemoteManager from module.database import DatabaseBackend, FileHandler -from module.utils import freeSpace, formatSize, get_console_encoding +import module.common.pylgettext as gettext +from module.utils import formatSize, get_console_encoding +from module.utils.fs import free_space, exists, makedirs, join from codecs import getwriter -enc = get_console_encoding(sys.stdout.encoding) +# test runner overwrites sys.stdout +if hasattr(sys.stdout, "encoding"): enc = get_console_encoding(sys.stdout.encoding) +else: enc = "utf8" + +sys._stdout = sys.stdout sys.stdout = getwriter(enc)(sys.stdout, errors="replace") # TODO List @@ -73,6 +82,7 @@ class Core(object): self.running = False self.daemon = False self.remote = True + self.pdb = None self.arg_links = [] self.pidfile = "pyload.pid" self.deleteLinks = False # will delete links on startup @@ -129,7 +139,7 @@ class Core(object): print pid exit(0) else: - print "false" + print "false" exit(1) elif option == "--clean": self.cleanTree() @@ -144,7 +154,7 @@ class Core(object): def print_help(self): print "" - print "pyLoad v%s 2008-2011 the pyLoad Team" % CURRENT_VERSION + print "pyLoad v%s 2008-2012 the pyLoad Team" % CURRENT_VERSION print "" if sys.argv[0].endswith(".py"): print "Usage: python pyLoadCore.py [options]" @@ -258,12 +268,12 @@ class Core(object): print join(path, f) remove(join(path, f)) - def start(self, rpc=True, web=True): + def start(self, rpc=True, web=True, tests=False): """ starts the fun :D """ self.version = CURRENT_VERSION - if not exists("pyload.conf"): + if not exists("pyload.conf") and not tests: from module.setup import Setup print "This is your first start, running configuration assistent now." @@ -295,11 +305,15 @@ class Core(object): languages=[self.config['general']['language'],"en"],fallback=True) translation.install(True) + # load again so translations are propagated + self.config.loadDefault() + self.debug = self.doDebug or self.config['general']['debug_mode'] self.remote &= self.config['remote']['activated'] pid = self.isAlreadyRunning() - if pid: + # dont exit when in test runner + if pid and not tests: print _("pyLoad already running with pid %s") % pid exit() @@ -326,8 +340,6 @@ class Core(object): except Exception, e: print _("Failed changing user: %s") % e - self.check_file(self.config['log']['log_folder'], _("folder for logs"), True) - if self.debug: self.init_logger(logging.DEBUG) # logging level else: @@ -339,8 +351,9 @@ class Core(object): self.log.info(_("Starting") + " pyLoad %s" % CURRENT_VERSION) self.log.info(_("Using home directory: %s") % getcwd()) - - self.writePidFile() + + if not tests: + self.writePidFile() #@TODO refractor @@ -348,24 +361,15 @@ class Core(object): self.log.debug("Remote activated: %s" % self.remote) self.check_install("Crypto", _("pycrypto to decode container files")) - #img = self.check_install("Image", _("Python Image Libary (PIL) for captcha reading")) - #self.check_install("pycurl", _("pycurl to download any files"), True, True) - self.check_file("tmp", _("folder for temporary files"), True) - #tesser = self.check_install("tesseract", _("tesseract for captcha reading"), False) if os.name != "nt" else True self.captcha = True # checks seems to fail, althoug tesseract is available - self.check_file(self.config['general']['download_folder'], _("folder for downloads"), True) - if self.config['ssl']['activated']: self.check_install("OpenSSL", _("OpenSSL for secure connection")) - self.setupDB() - if self.config.oldRemoteData: - self.log.info(_("Moving old user config to DB")) - self.db.addUser(self.config.oldRemoteData["username"], self.config.oldRemoteData["password"]) - self.log.info(_("Please check your logindata with ./pyLoadCore.py -u")) + self.eventManager = EventManager(self) + self.setupDB() if self.deleteLinks: self.log.info(_("All links removed")) @@ -379,7 +383,7 @@ class Core(object): # later imported because they would trigger api import, and remote value not set correctly from module import Api from module.HookManager import HookManager - from module.ThreadManager import ThreadManager + from module.threads.ThreadManager import ThreadManager if Api.activated != self.remote: self.log.warning("Import error: API remote status not correct.") @@ -390,7 +394,7 @@ class Core(object): #hell yeah, so many important managers :D self.pluginManager = PluginManager(self) - self.pullManager = PullManager(self) + self.interActionManager = None #stub self.accountManager = AccountManager(self) self.threadManager = ThreadManager(self) self.captchaManager = CaptchaManager(self) @@ -399,6 +403,9 @@ class Core(object): self.js = JsEngine() + # enough initialization for test cases + if tests: return + self.log.info(_("Downloadtime: %s") % self.api.isTimeDownload()) if rpc: @@ -407,7 +414,12 @@ class Core(object): if web: self.init_webserver() - spaceLeft = freeSpace(self.config["general"]["download_folder"]) + dl_folder = self.config["general"]["download_folder"] + + if not exists(dl_folder): + makedirs(dl_folder) + + spaceLeft = free_space(dl_folder) self.log.info(_("Free space: %s") % formatSize(spaceLeft)) @@ -430,15 +442,14 @@ class Core(object): #self.scheduler.addJob(0, self.accountManager.getAccountInfos) self.log.info(_("Activating Accounts...")) - self.accountManager.getAccountInfos() - + self.accountManager.refreshAllAccounts() self.threadManager.pause = False self.running = True - self.log.info(_("Activating Plugins...")) - self.hookManager.coreReady() + self.hookManager.activateHooks() self.log.info(_("pyLoad is up and running")) + self.eventManager.dispatchEvent("coreReady") #test api # from module.common.APIExerciser import startApiExerciser @@ -447,10 +458,13 @@ class Core(object): #some memory stats # from guppy import hpy # hp=hpy() +# print hp.heap() # import objgraph -# objgraph.show_most_common_types(limit=20) +# objgraph.show_most_common_types(limit=30) # import memdebug # memdebug.start(8002) +# from meliae import scanner +# scanner.dump_all_objects(self.path('objs.json')) locals().clear() @@ -486,6 +500,9 @@ class Core(object): console.setFormatter(frm) self.log = logging.getLogger("log") # settable in config + if not exists(self.config['log']['log_folder']): + makedirs(self.config['log']['log_folder'], 0600) + if self.config['log']['file_log']: if self.config['log']['log_rotate']: file_handler = logging.handlers.RotatingFileHandler(join(self.config['log']['log_folder'], 'log.txt'), @@ -523,43 +540,6 @@ class Core(object): return False - def check_file(self, check_names, description="", folder=False, empty=True, essential=False, quiet=False): - """check wether needed files exists""" - tmp_names = [] - if not type(check_names) == list: - tmp_names.append(check_names) - else: - tmp_names.extend(check_names) - file_created = True - file_exists = True - for tmp_name in tmp_names: - if not exists(tmp_name): - file_exists = False - if empty: - try: - if folder: - tmp_name = tmp_name.replace("/", sep) - makedirs(tmp_name) - else: - open(tmp_name, "w") - except: - file_created = False - else: - file_created = False - - if not file_exists and not quiet: - if file_created: - #self.log.info( _("%s created") % description ) - pass - else: - if not empty: - self.log.warning( - _("could not find %(desc)s: %(name)s") % {"desc": description, "name": tmp_name}) - else: - print _("could not create %(desc)s: %(name)s") % {"desc": description, "name": tmp_name} - if essential: - exit() - def isClientConnected(self): return (self.lastClientConnected + 30) > time() @@ -578,10 +558,13 @@ class Core(object): def shutdown(self): self.log.info(_("shutting down...")) + self.eventManager.dispatchEvent("coreShutdown") try: if self.config['webinterface']['activated'] and hasattr(self, "webserver"): self.webserver.quit() + + for thread in self.threadManager.threads: thread.put("quit") pyfiles = self.files.cache.values() @@ -589,7 +572,7 @@ class Core(object): for pyfile in pyfiles: pyfile.abortDownload() - self.hookManager.coreExiting() + self.hookManager.deactivateHooks() except: if self.debug: @@ -602,11 +585,23 @@ class Core(object): self.deletePidFile() + def shell(self): + """ stop and open a ipython shell inplace""" + if self.debug: + from IPython import embed + sys.stdout = sys._stdout + embed() + + def breakpoint(self): + if self.debug: + from IPython.core.debugger import Pdb + sys.stdout = sys._stdout + if not self.pdb: self.pdb = Pdb() + self.pdb.set_trace() def path(self, *args): return join(pypath, *args) - def deamon(): try: pid = os.fork() diff --git a/pyLoadGui.py b/pyLoadGui.py deleted file mode 100755 index 5f620e52a..000000000 --- a/pyLoadGui.py +++ /dev/null @@ -1,765 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay -""" - -import sys - -from uuid import uuid4 as uuid # should be above PyQt imports -from time import sleep, time - -from base64 import b64decode - -from PyQt4.QtCore import * -from PyQt4.QtGui import * - -import re -import module.common.pylgettext as gettext -import os -from os.path import abspath -from os.path import join -from os.path import basename -from os.path import commonprefix - -from module import InitHomeDir -from module.gui.ConnectionManager import * -from module.gui.Connector import Connector -from module.gui.MainWindow import * -from module.gui.Queue import * -from module.gui.Overview import * -from module.gui.Collector import * -from module.gui.XMLParser import * -from module.gui.CoreConfigParser import ConfigParser - -from module.lib.rename_process import renameProcess -from module.utils import formatSize, formatSpeed - -try: - import pynotify -except ImportError: - print "pynotify not installed, falling back to qt tray notification" - -class main(QObject): - def __init__(self): - """ - main setup - """ - QObject.__init__(self) - self.app = QApplication(sys.argv) - self.path = pypath - self.homedir = abspath("") - - self.configdir = "" - - self.init(True) - - def init(self, first=False): - """ - set main things up - """ - self.parser = XMLParser(join(self.configdir, "gui.xml"), join(self.path, "module", "config", "gui_default.xml")) - lang = self.parser.xml.elementsByTagName("language").item(0).toElement().text() - if not lang: - parser = XMLParser(join(self.path, "module", "config", "gui_default.xml")) - lang = parser.xml.elementsByTagName("language").item(0).toElement().text() - - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("pyLoadGui", join(pypath, "locale"), languages=[str(lang), "en"], fallback=True) - try: - translation.install(unicode=(True if sys.stdout.encoding.lower().startswith("utf") else False)) - except: - translation.install(unicode=False) - - - self.connector = Connector() - self.mainWindow = MainWindow(self.connector) - self.connWindow = ConnectionManager() - self.mainloop = self.Loop(self) - self.connectSignals() - - self.checkClipboard = False - default = self.refreshConnections() - self.connData = None - self.captchaProcessing = False - self.serverStatus = {"freespace":0} - - self.core = None # pyLoadCore if started - self.connectionLost = False - - if True: # when used if first, minimizing not working correctly.. - self.tray = TrayIcon() - self.tray.show() - self.notification = Notification(self.tray) - self.connect(self, SIGNAL("showMessage"), self.notification.showMessage) - self.connect(self.tray.exitAction, SIGNAL("triggered()"), self.slotQuit) - self.connect(self.tray.showAction, SIGNAL("toggled(bool)"), self.mainWindow.setVisible) - self.connect(self.mainWindow, SIGNAL("hidden"), self.tray.mainWindowHidden) - - if not first: - self.connWindow.show() - else: - self.connWindow.edit.setData(default) - data = self.connWindow.edit.getData() - self.slotConnect(data) - - def startMain(self): - """ - start all refresh threads and show main window - """ - if not self.connector.connectProxy(): - self.init() - return - self.connect(self.connector, SIGNAL("connectionLost"), self.slotConnectionLost) - self.restoreMainWindow() - self.mainWindow.show() - self.initQueue() - self.initPackageCollector() - self.mainloop.start() - self.clipboard = self.app.clipboard() - self.connect(self.clipboard, SIGNAL('dataChanged()'), self.slotClipboardChange) - self.mainWindow.actions["clipboard"].setChecked(self.checkClipboard) - - self.mainWindow.tabs["settings"]["w"].setConnector(self.connector) - self.mainWindow.tabs["settings"]["w"].loadConfig() - self.tray.showAction.setDisabled(False) - - def stopMain(self): - """ - stop all refresh threads and hide main window - """ - self.tray.showAction.setDisabled(True) - self.disconnect(self.clipboard, SIGNAL('dataChanged()'), self.slotClipboardChange) - self.disconnect(self.connector, SIGNAL("connectionLost"), self.slotConnectionLost) - self.mainloop.stop() - self.mainWindow.saveWindow() - self.mainWindow.hide() - self.queue.stop() - - def connectSignals(self): - """ - signal and slot stuff, yay! - """ - self.connect(self.connector, SIGNAL("errorBox"), self.slotErrorBox) - self.connect(self.connWindow, SIGNAL("saveConnection"), self.slotSaveConnection) - self.connect(self.connWindow, SIGNAL("removeConnection"), self.slotRemoveConnection) - self.connect(self.connWindow, SIGNAL("connect"), self.slotConnect) - self.connect(self.mainWindow, SIGNAL("connector"), self.slotShowConnector) - self.connect(self.mainWindow, SIGNAL("addPackage"), self.slotAddPackage) - self.connect(self.mainWindow, SIGNAL("setDownloadStatus"), self.slotSetDownloadStatus) - self.connect(self.mainWindow, SIGNAL("saveMainWindow"), self.slotSaveMainWindow) - self.connect(self.mainWindow, SIGNAL("pushPackageToQueue"), self.slotPushPackageToQueue) - self.connect(self.mainWindow, SIGNAL("restartDownload"), self.slotRestartDownload) - self.connect(self.mainWindow, SIGNAL("removeDownload"), self.slotRemoveDownload) - self.connect(self.mainWindow, SIGNAL("abortDownload"), self.slotAbortDownload) - self.connect(self.mainWindow, SIGNAL("addContainer"), self.slotAddContainer) - self.connect(self.mainWindow, SIGNAL("stopAllDownloads"), self.slotStopAllDownloads) - self.connect(self.mainWindow, SIGNAL("setClipboardStatus"), self.slotSetClipboardStatus) - self.connect(self.mainWindow, SIGNAL("changePackageName"), self.slotChangePackageName) - self.connect(self.mainWindow, SIGNAL("pullOutPackage"), self.slotPullOutPackage) - self.connect(self.mainWindow, SIGNAL("refreshStatus"), self.slotRefreshStatus) - self.connect(self.mainWindow, SIGNAL("reloadAccounts"), self.slotReloadAccounts) - self.connect(self.mainWindow, SIGNAL("Quit"), self.slotQuit) - - self.connect(self.mainWindow.mactions["exit"], SIGNAL("triggered()"), self.slotQuit) - self.connect(self.mainWindow.captchaDock, SIGNAL("done"), self.slotCaptchaDone) - - def slotShowConnector(self): - """ - emitted from main window (menu) - hide the main window and show connection manager - (to switch to other core) - """ - self.quitInternal() - self.stopMain() - self.init() - - #def quit(self): #not used anymore? - # """ - # quit gui - # """ - # self.app.quit() - - def loop(self): - """ - start application loop - """ - sys.exit(self.app.exec_()) - - def slotErrorBox(self, msg): - """ - display a nice error box - """ - msgb = QMessageBox(QMessageBox.Warning, "Error", msg) - msgb.exec_() - - def initPackageCollector(self): - """ - init the package collector view - * columns - * selection - * refresh thread - * drag'n'drop - """ - view = self.mainWindow.tabs["collector"]["package_view"] - view.setSelectionBehavior(QAbstractItemView.SelectRows) - view.setSelectionMode(QAbstractItemView.ExtendedSelection) - def dropEvent(klass, event): - event.setDropAction(Qt.CopyAction) - event.accept() - view = event.source() - if view == klass: - items = view.selectedItems() - for item in items: - if not hasattr(item.parent(), "getPackData"): - continue - target = view.itemAt(event.pos()) - if not hasattr(target, "getPackData"): - target = target.parent() - klass.emit(SIGNAL("droppedToPack"), target.getPackData()["id"], item.getFileData()["id"]) - event.ignore() - return - items = view.selectedItems() - for item in items: - row = view.indexOfTopLevelItem(item) - view.takeTopLevelItem(row) - def dragEvent(klass, event): - #view = event.source() - #dragOkay = False - #items = view.selectedItems() - #for item in items: - # if hasattr(item, "_data"): - # if item._data["id"] == "fixed" or item.parent()._data["id"] == "fixed": - # dragOkay = True - # else: - # dragOkay = True - #if dragOkay: - event.accept() - #else: - # event.ignore() - view.dropEvent = dropEvent - view.dragEnterEvent = dragEvent - view.setDragEnabled(True) - view.setDragDropMode(QAbstractItemView.DragDrop) - view.setDropIndicatorShown(True) - view.setDragDropOverwriteMode(True) - view.connect(view, SIGNAL("droppedToPack"), self.slotAddFileToPackage) - #self.packageCollector = PackageCollector(view, self.connector) - self.packageCollector = view.model() - - def initQueue(self): - """ - init the queue view - * columns - * progressbar - """ - view = self.mainWindow.tabs["queue"]["view"] - view.setSelectionBehavior(QAbstractItemView.SelectRows) - view.setSelectionMode(QAbstractItemView.ExtendedSelection) - self.queue = view.model() - self.connect(self.queue, SIGNAL("updateCount"), self.slotUpdateCount) - overview = self.mainWindow.tabs["overview"]["view"].model() - overview.queue = self.queue - self.connect(self.queue, SIGNAL("updateCount"), overview.queueChanged) - self.queue.start() - - def slotUpdateCount(self, pc, fc): - self.mainWindow.packageCount.setText("%i" % pc) - self.mainWindow.fileCount.setText("%i" % fc) - - def refreshServerStatus(self): - """ - refresh server status and overall speed in the status bar - """ - s = self.connector.statusServer() - if s.pause: - self.mainWindow.status.setText(_("paused")) - else: - self.mainWindow.status.setText(_("running")) - self.mainWindow.speed.setText(formatSpeed(s.speed)) - self.mainWindow.space.setText(formatSize(self.serverStatus["freespace"])) - self.mainWindow.actions["toggle_status"].setChecked(not s.pause) - - def refreshLog(self): - """ - update log window - """ - offset = self.mainWindow.tabs["log"]["text"].logOffset - lines = self.connector.getLog(offset) - if not lines: - return - self.mainWindow.tabs["log"]["text"].logOffset += len(lines) - for line in lines: - self.mainWindow.tabs["log"]["text"].emit(SIGNAL("append(QString)"), line.strip("\n")) - cursor = self.mainWindow.tabs["log"]["text"].textCursor() - cursor.movePosition(QTextCursor.End, QTextCursor.MoveAnchor) - self.mainWindow.tabs["log"]["text"].setTextCursor(cursor) - - def getConnections(self): - """ - parse all connections in the config file - """ - connectionsNode = self.parser.xml.elementsByTagName("connections").item(0) - if connectionsNode.isNull(): - raise Exception("null") - connections = self.parser.parseNode(connectionsNode) - ret = [] - for conn in connections: - data = {} - data["type"] = conn.attribute("type", "remote") - data["default"] = conn.attribute("default", "False") - data["id"] = conn.attribute("id", uuid().hex) - if data["default"] == "True": - data["default"] = True - else: - data["default"] = False - subs = self.parser.parseNode(conn, "dict") - if not subs.has_key("name"): - data["name"] = _("Unnamed") - else: - data["name"] = subs["name"].text() - if data["type"] == "remote": - if not subs.has_key("server"): - continue - else: - data["host"] = subs["server"].text() - data["user"] = subs["server"].attribute("user", "admin") - data["port"] = int(subs["server"].attribute("port", "7227")) - data["password"] = subs["server"].attribute("password", "") - ret.append(data) - return ret - - def slotSaveConnection(self, data): - """ - save connection to config file - """ - connectionsNode = self.parser.xml.elementsByTagName("connections").item(0) - if connectionsNode.isNull(): - raise Exception("null") - connections = self.parser.parseNode(connectionsNode) - connNode = self.parser.xml.createElement("connection") - connNode.setAttribute("default", str(data["default"])) - connNode.setAttribute("type", data["type"]) - connNode.setAttribute("id", data["id"]) - nameNode = self.parser.xml.createElement("name") - nameText = self.parser.xml.createTextNode(data["name"]) - nameNode.appendChild(nameText) - connNode.appendChild(nameNode) - if data["type"] == "remote": - serverNode = self.parser.xml.createElement("server") - serverNode.setAttribute("user", data["user"]) - serverNode.setAttribute("port", data["port"]) - serverNode.setAttribute("password", data["password"]) - hostText = self.parser.xml.createTextNode(data["host"]) - serverNode.appendChild(hostText) - connNode.appendChild(serverNode) - found = False - for c in connections: - cid = c.attribute("id", "None") - if str(cid) == str(data["id"]): - found = c - break - if found: - connectionsNode.replaceChild(connNode, found) - else: - connectionsNode.appendChild(connNode) - self.parser.saveData() - self.refreshConnections() - - def slotRemoveConnection(self, data): - """ - remove connection from config file - """ - connectionsNode = self.parser.xml.elementsByTagName("connections").item(0) - if connectionsNode.isNull(): - raise Exception("null") - connections = self.parser.parseNode(connectionsNode) - found = False - for c in connections: - cid = c.attribute("id", "None") - if str(cid) == str(data["id"]): - found = c - break - if found: - connectionsNode.removeChild(found) - self.parser.saveData() - self.refreshConnections() - - def slotConnect(self, data): - """ - connect to a core - if connection is local, parse the core config file for data - if internal, start pyLoadCore - set up connector, show main window - """ - self.connWindow.hide() - if data["type"] not in ("remote","internal"): - - coreparser = ConfigParser(self.configdir) - if not coreparser.config: - self.connector.setConnectionData("127.0.0.1", 7227, "anonymous", "anonymous", False) - else: - self.connector.setConnectionData("127.0.0.1", coreparser.get("remote","port"), "anonymous", "anonymous") - - elif data["type"] == "remote": - self.connector.setConnectionData(data["host"], data["port"], data["user"], data["password"]) - - elif data["type"] == "internal": - from pyLoadCore import Core - from module.ConfigParser import ConfigParser as CoreConfig - import thread - - if not self.core: - - config = CoreConfig() #create so at least default config exists - self.core = Core() - self.core.startedInGui = True - thread.start_new_thread(self.core.start, (False, False)) - while not self.core.running: - sleep(0.5) - - self.connector.proxy = self.core.api - self.connector.internal = True - - #self.connector.setConnectionData("127.0.0.1", config.get("remote","port"), "anonymous", "anonymous") - - self.startMain() -# try: -# host = data["host"] -# except: -# host = "127.0.0.1" - - def refreshConnections(self): - """ - reload connetions and display them - """ - self.parser.loadData() - conns = self.getConnections() - self.connWindow.emit(SIGNAL("setConnections"), conns) - for conn in conns: - if conn["default"]: - return conn - return None - - def slotSetDownloadStatus(self, status): - """ - toolbar start/pause slot - """ - if status: - self.connector.unpauseServer() - else: - self.connector.pauseServer() - - def slotAddPackage(self, name, links, password=None): - """ - emitted from main window - add package to the collector - """ - pack = self.connector.addPackage(name, links, Destination.Collector) - if password: - data = {"password": password} - self.connector.setPackageData(pack, data) - - def slotAddFileToPackage(self, pid, fid): #TODO deprecated? gets called - """ - emitted from collector view after a drop action - """ - #self.connector.addFileToPackage(fid, pid) - pass - - def slotAddContainer(self, path): - """ - emitted from main window - add container - """ - filename = basename(path) - #type = "".join(filename.split(".")[-1]) - fh = open(path, "r") - content = fh.read() - fh.close() - self.connector.uploadContainer(filename, content) - - def slotSaveMainWindow(self, state, geo): - """ - save the window geometry and toolbar/dock position to config file - """ - mainWindowNode = self.parser.xml.elementsByTagName("mainWindow").item(0) - if mainWindowNode.isNull(): - mainWindowNode = self.parser.xml.createElement("mainWindow") - self.parser.root.appendChild(mainWindowNode) - stateNode = mainWindowNode.toElement().elementsByTagName("state").item(0) - geoNode = mainWindowNode.toElement().elementsByTagName("geometry").item(0) - newStateNode = self.parser.xml.createTextNode(state) - newGeoNode = self.parser.xml.createTextNode(geo) - - stateNode.removeChild(stateNode.firstChild()) - geoNode.removeChild(geoNode.firstChild()) - stateNode.appendChild(newStateNode) - geoNode.appendChild(newGeoNode) - - self.parser.saveData() - - def restoreMainWindow(self): - """ - load and restore main window geometry and toolbar/dock position from config - """ - mainWindowNode = self.parser.xml.elementsByTagName("mainWindow").item(0) - if mainWindowNode.isNull(): - return - nodes = self.parser.parseNode(mainWindowNode, "dict") - - state = str(nodes["state"].text()) - geo = str(nodes["geometry"].text()) - - self.mainWindow.restoreWindow(state, geo) - self.mainWindow.captchaDock.hide() - - def slotPushPackageToQueue(self, id): - """ - emitted from main window - push the collector package to queue - """ - self.connector.pushToQueue(id) - - def slotRestartDownload(self, id, isPack): - """ - emitted from main window - restart download - """ - if isPack: - self.connector.restartPackage(id) - else: - self.connector.restartFile(id) - - def slotRefreshStatus(self, id): - """ - emitted from main window - refresh download status - """ - self.connector.recheckPackage(id) - - def slotRemoveDownload(self, id, isPack): - """ - emitted from main window - remove download - """ - if isPack: - self.connector.deletePackages([id]) - else: - self.connector.deleteFiles([id]) - - def slotAbortDownload(self, id, isPack): - """ - emitted from main window - remove download - """ - if isPack: - data = self.connector.getFileOrder(id) #less data to transmit - self.connector.stopDownloads(data.values()) - else: - self.connector.stopDownloads([id]) - - def slotStopAllDownloads(self): - """ - emitted from main window - stop all running downloads - """ - self.connector.stopAllDownloads() - - def slotClipboardChange(self): - """ - called if clipboard changes - """ - if self.checkClipboard: - text = self.clipboard.text() - pattern = re.compile(r"(http|https|ftp)://[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(([0-9]{1,5})?/.*)?") - matches = pattern.finditer(text) - - # thanks to: jmansour //#139 - links = [str(match.group(0)) for match in matches] - if len(links) == 0: - return - - filenames = [link.rpartition("/")[2] for link in links] - - packagename = commonprefix(filenames) - if len(packagename) == 0: - packagename = filenames[0] - - self.slotAddPackage(packagename, links) - - def slotSetClipboardStatus(self, status): - """ - set clipboard checking - """ - self.checkClipboard = status - - def slotChangePackageName(self, pid, name): - """ - package name edit finished - """ - self.connector.setPackageName(pid, str(name)) - - def slotPullOutPackage(self, pid): - """ - pull package out of the queue - """ - self.connector.pullFromQueue(pid) - - def checkCaptcha(self): - if self.connector.isCaptchaWaiting() and self.mainWindow.captchaDock.isFree(): - t = self.connector.getCaptchaTask(False) - self.mainWindow.show() - self.mainWindow.raise_() - self.mainWindow.activateWindow() - self.mainWindow.captchaDock.emit(SIGNAL("setTask"), t.tid, b64decode(t.data), t.type) - elif not self.mainWindow.captchaDock.isFree(): - status = self.connector.getCaptchaTaskStatus(self.mainWindow.captchaDock.currentID) - if not (status == "user" or status == "shared-user"): - self.mainWindow.captchaDock.hide() - self.mainWindow.captchaDock.processing = False - self.mainWindow.captchaDock.currentID = None - - def slotCaptchaDone(self, cid, result): - self.connector.setCaptchaResult(cid, str(result)) - - def pullEvents(self): - events = self.connector.getEvents(self.connector.connectionID) - if not events: - return - for event in events: - if event.eventname == "account": - self.mainWindow.emit(SIGNAL("reloadAccounts"), False) - elif event.eventname == "config": - pass - elif event.destination == Destination.Queue: - self.queue.addEvent(event) - try: - if event.eventname == "update" and event.type == ElementType.File: - info = self.connector.getFileData(event.id) - if info.statusmsg == "finished": - self.emit(SIGNAL("showMessage"), _("Finished downloading of '%s'") % info.name) - elif info.statusmsg == "failed": - self.emit(SIGNAL("showMessage"), _("Failed downloading '%s'!") % info.name) - if event.event == "insert" and event.type == ElementType.File: - info = self.connector.getLinkInfo(event[3]) - self.emit(SIGNAL("showMessage"), _("Added '%s' to queue") % info.name) - except: - print "can't send notification" - elif event.destination == Destination.Collector: - self.packageCollector.addEvent(event) - - def slotReloadAccounts(self, force=False): - self.mainWindow.tabs["accounts"]["view"].model().reloadData(force) - - def slotQuit(self): - self.tray.hide() - self.quitInternal() - self.app.quit() - - def quitInternal(self): - if self.core: - self.core.api.kill() - for i in range(10): - if self.core.shuttedDown: - break - sleep(0.5) - - def slotConnectionLost(self): - if not self.connectionLost: - self.connectionLost = True - m = QMessageBox(QMessageBox.Critical, _("Connection lost"), _("Lost connection to the core!"), QMessageBox.Ok) - m.exec_() - self.slotQuit() - - class Loop(): - def __init__(self, parent): - self.parent = parent - self.timer = QTimer() - self.timer.connect(self.timer, SIGNAL("timeout()"), self.update) - self.lastSpaceCheck = 0 - - def start(self): - self.update() - self.timer.start(1000) - - def update(self): - """ - methods to call - """ - self.parent.refreshServerStatus() - if self.lastSpaceCheck + 5 < time(): - self.lastSpaceCheck = time() - self.parent.serverStatus["freespace"] = self.parent.connector.freeSpace() - self.parent.refreshLog() - self.parent.checkCaptcha() - self.parent.pullEvents() - - def stop(self): - self.timer.stop() - - -class TrayIcon(QSystemTrayIcon): - def __init__(self): - QSystemTrayIcon.__init__(self, QIcon(join(pypath, "icons", "logo-gui.png"))) - self.contextMenu = QMenu() - self.showAction = QAction(_("Show"), self.contextMenu) - self.showAction.setCheckable(True) - self.showAction.setChecked(True) - self.showAction.setDisabled(True) - self.contextMenu.addAction(self.showAction) - self.exitAction = QAction(QIcon(join(pypath, "icons", "close.png")), _("Exit"), self.contextMenu) - self.contextMenu.addAction(self.exitAction) - self.setContextMenu(self.contextMenu) - - self.connect(self, SIGNAL("activated(QSystemTrayIcon::ActivationReason)"), self.clicked) - - def mainWindowHidden(self): - self.showAction.setChecked(False) - - def clicked(self, reason): - if self.showAction.isEnabled(): - if reason == QSystemTrayIcon.Trigger: - self.showAction.toggle() - -class Notification(QObject): - def __init__(self, tray): - QObject.__init__(self) - self.tray = tray - self.usePynotify = False - - try: - self.usePynotify = pynotify.init("icon-summary-body") - except: - print "init error" - - def showMessage(self, body): - if self.usePynotify: - n = pynotify.Notification("pyload", body, join(pypath, "icons", "logo.png")) - try: - n.set_hint_string("x-canonical-append", "") - except: - pass - n.show() - else: - self.tray.showMessage("pyload", body) - -if __name__ == "__main__": - renameProcess('pyLoadGui') - app = main() - app.loop() - diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..d04135803 --- /dev/null +++ b/setup.py @@ -0,0 +1,7 @@ +import os +if os.path.exists("paver-minilib.zip"): + import sys + sys.path.insert(0, "paver-minilib.zip") + +import paver.tasks +paver.tasks.main() diff --git a/systemCheck.py b/systemCheck.py index 60fe0313b..b16704ac9 100644 --- a/systemCheck.py +++ b/systemCheck.py @@ -94,24 +94,6 @@ def main(): for line in core_info: print(line) - - print("\n## pyLoadGui ##") - - gui_err = [] - - try: - import PyQt4 - except: - gui_err.append("GUI won't work without pyqt4 !!") - - if gui_err: - print("The system check has detected some errors:\n") - for err in gui_err: - print(err) - else: - print("No Problems detected, pyLoadGui should work fine.") - - print("\n## Webinterface ##") web_err = [] @@ -139,4 +121,5 @@ def main(): if __name__ == "__main__": main() - raw_input("Press Enter to Exit.") + # comp. with py2 and 3 + input("Press Enter to Exit.") diff --git a/testlinks.txt b/testlinks.txt deleted file mode 100644 index 428cf63ea..000000000 --- a/testlinks.txt +++ /dev/null @@ -1,26 +0,0 @@ -sha1: - 961486646bf3c1d5d7a45ec32bb62e1bc4f2d894 random.bin -md5: - d76505d0869f9f928a17d42d66326307 random.bin - -please save bandwith on our server, -use this link only for remote uploads to new hoster -http://get.pyload.org/static/random.bin ---------------------------------------- -http://netload.in/datei9XirAJZs79/random.bin.htm -http://ul.to/file/o41isx -http://rapidshare.com/files/445996776/random.bin -http://dl.free.fr/d4aL5dyXY -http://www.duckload.com/dl/QggW2 -http://files.mail.ru/32EW66 -http://www.fileserve.com/file/MxjZXjX -http://www.4shared.com/file/-O5CBhQV/random.html -http://hotfile.com/dl/101569859/2e01f04/random.bin.html -http://www.megaupload.com/?d=1JZLOP3B -http://www.share.cx/files/235687689252/random.bin.html -http://www.share-online.biz/download.php?id=PTCOX1GL6XAH -http://www.shragle.com/files/f899389b/random.bin -http://www10.zippyshare.com/v/76557688/file.html -http://yourfiles.to/?d=312EC6E911 -http://depositfiles.com/files/k8la98953 -http://uploading.com/files/3896f5a1/random.bin/ diff --git a/tests/CrypterPluginTester.py b/tests/CrypterPluginTester.py new file mode 100644 index 000000000..67e5ddebc --- /dev/null +++ b/tests/CrypterPluginTester.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +from os.path import dirname, join +from nose.tools import nottest + +from logging import log, DEBUG + +from helper.Stubs import Core +from helper.PluginTester import PluginTester + +from module.plugins.Base import Fail +from module.utils import accumulate, to_int + +class CrypterPluginTester(PluginTester): + @nottest + def test_plugin(self, name, url, flag): + + print "%s: %s" % (name, url.encode("utf8")) + log(DEBUG, "%s: %s", name, url.encode("utf8")) + + plugin = self.core.pluginManager.getPluginClass(name) + p = plugin(self.core, None, "") + self.thread.plugin = p + + try: + result = p._decrypt([url]) + + if to_int(flag): + assert to_int(flag) == len(result) + + except Exception, e: + if isinstance(e, Fail) and flag == "fail": + pass + else: + raise + + +# setup methods + +c = Core() + +f = open(join(dirname(__file__), "crypterlinks.txt")) +links = [x.strip() for x in f.readlines()] +urls = [] +flags = {} + +for l in links: + if not l or l.startswith("#"): continue + if l.startswith("http"): + if "||" in l: + l, flag = l.split("||") + flags[l] = flag + + urls.append(l) + +h, crypter = c.pluginManager.parseUrls(urls) +plugins = accumulate(crypter) +for plugin, urls in plugins.iteritems(): + + def meta_class(plugin): + class _testerClass(CrypterPluginTester): + pass + _testerClass.__name__ = plugin + return _testerClass + + _testerClass = meta_class(plugin) + + for i, url in enumerate(urls): + def meta(plugin, url, flag, sig): + def _test(self): + self.test_plugin(plugin, url, flag) + + _test.func_name = sig + return _test + + sig = "test_LINK%d" % i + setattr(_testerClass, sig, meta(plugin, url, flags.get(url, None), sig)) + print url + + locals()[plugin] = _testerClass + del _testerClass diff --git a/tests/HosterPluginTester.py b/tests/HosterPluginTester.py new file mode 100644 index 000000000..f8a400c6d --- /dev/null +++ b/tests/HosterPluginTester.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- + +from os.path import dirname +from logging import log, DEBUG +from hashlib import md5 +from time import time +from shutil import move +import codecs + +from nose.tools import nottest + +from helper.Stubs import Core +from helper.PluginTester import PluginTester + +from module.PyFile import PyFile +from module.plugins.Base import Fail +from module.utils import accumulate +from module.utils.fs import save_join, join, exists, listdir, remove, stat, fs_encode + +DL_DIR = join("Downloads", "tmp") + +class HosterPluginTester(PluginTester): + files = {} + + def setUp(self): + PluginTester.setUp(self) + for f in self.files: + if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f)) + + # folder for reports + report = join("tmp", self.__class__.__name__) + if exists(report): + for f in listdir(report): + remove(join(report, f)) + + + @nottest + def test_plugin(self, name, url, flag): + # Print to stdout to see whats going on + print "%s: %s, %s" % (name, url.encode("utf8"), flag) + log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag) + + # url and plugin should be only important thing + pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0) + pyfile.initPlugin() + + self.thread.pyfile = pyfile + self.thread.plugin = pyfile.plugin + + try: + a = time() + pyfile.plugin.preprocessing(self.thread) + + log(DEBUG, "downloading took %ds" % (time() - a)) + log(DEBUG, "size %d kb" % (pyfile.size / 1024)) + + if flag == "offline": + raise Exception("No offline Exception raised.") + + if pyfile.name not in self.files: + raise Exception("Filename %s not recognized." % pyfile.name) + + if not exists(save_join(DL_DIR, pyfile.name)): + raise Exception("File %s does not exists." % pyfile.name) + + hash = md5() + f = open(save_join(DL_DIR, pyfile.name), "rb") + while True: + buf = f.read(4096) + if not buf: break + hash.update(buf) + f.close() + + if hash.hexdigest() != self.files[pyfile.name]: + log(DEBUG, "Hash is %s" % hash.hexdigest()) + + size = stat(f.name).st_size + if size < 1024 * 1024 * 10: # 10MB + # Copy for debug report + move(fs_encode(f.name), fs_encode(join("tmp", plugin, f.name))) + + raise Exception("Hash does not match.") + + + + except Exception, e: + if isinstance(e, Fail) and flag == "fail": + pass + elif isinstance(e, Fail) and flag == "offline" and e.message == "offline": + pass + else: + raise + + +# setup methods + +c = Core() + +# decode everything as unicode +f = codecs.open(join(dirname(__file__), "hosterlinks.txt"), "r", "utf_8") +links = [x.strip() for x in f.readlines()] +urls = [] +flags = {} + +for l in links: + if not l or l.startswith("#"): continue + if l.startswith("http"): + if "||" in l: + l, flag = l.split("||") + flags[l] = str(flag.strip()) + urls.append(l) + + elif len(l.rsplit(" ", 1)) == 2: + name, hash = l.rsplit(" ", 1) + HosterPluginTester.files[name] = str(hash) + +hoster, c = c.pluginManager.parseUrls(urls) + +plugins = accumulate(hoster) +for plugin, urls in plugins.iteritems(): + # closure functions to retain local scope + def meta_class(plugin): + class _testerClass(HosterPluginTester): + pass + _testerClass.__name__ = plugin + return _testerClass + + _testerClass = meta_class(plugin) + + for i, url in enumerate(urls): + def meta(__plugin, url, flag, sig): + def _test(self): + self.test_plugin(__plugin, url, flag) + + _test.func_name = sig + return _test + + tmp_flag = flags.get(url, None) + if flags.get(url, None): + sig = "test_LINK%d_%s" % (i, tmp_flag) + else: + sig = "test_LINK%d" % i + + # set test method + setattr(_testerClass, sig, meta(plugin, url, tmp_flag, sig)) + + + #register class + locals()[plugin] = _testerClass + # remove from locals, or tested twice + del _testerClass diff --git a/tests/clonedigger.sh b/tests/clonedigger.sh new file mode 100755 index 000000000..93c1cb323 --- /dev/null +++ b/tests/clonedigger.sh @@ -0,0 +1,2 @@ +#!/bin/bash +clonedigger -o cpd.xml --cpd-output --fast --ignore-dir=lib --ignore-dir=remote module diff --git a/tests/config/db.version b/tests/config/db.version new file mode 100644 index 000000000..bf0d87ab1 --- /dev/null +++ b/tests/config/db.version @@ -0,0 +1 @@ +4
\ No newline at end of file diff --git a/tests/config/plugin.conf b/tests/config/plugin.conf new file mode 100644 index 000000000..5e7ee3858 --- /dev/null +++ b/tests/config/plugin.conf @@ -0,0 +1,138 @@ +version: 2 + +[MultiuploadCom] +preferedHoster = multiupload +ignoredHoster = + +[SerienjunkiesOrg] +preferredHoster = RapidshareCom,UploadedTo,NetloadIn,FilefactoryCom,FreakshareNet,FilebaseTo,MegauploadCom,HotfileCom,DepositfilesCom,EasyshareCom,KickloadCom +changeName = True + +[EmbeduploadCom] +preferedHoster = embedupload +ignoredHoster = + +[MultiloadCz] +usedHoster = +ignoredHoster = + +[WiiReloadedOrg] +changeName = True + +[Xdcc] +nick = pyload +ident = pyloadident +realname = pyloadreal + +[UlozTo] +reuseCaptcha = True +captchaUser = +captchaNb = + +[YoutubeCom] +quality = hd +fmt = 0 +.mp4 = True +.flv = True +.webm = False +.3gp = False + +[RapidshareCom] +server = None + +[VeehdCom] +filename_spaces = False +replacement_char = _ + +[RealdebridCom] +https = False + +[ClickAndLoad] +activated = True +extern = False + +[ExtractArchive] +activated = True +fullpath = True +overwrite = True +passwordfile = unrar_passwords.txt +deletearchive = False +subfolder = False +destination = +queue = True +renice = 0 + +[CaptchaTrader] +activated = True +username = +force = False +passkey = + +[MergeFiles] +activated = False + +[IRCInterface] +activated = False +host = Enter your server here! +port = 6667 +ident = pyload-irc +realname = pyload-irc +nick = pyLoad-IRC +owner = Enter your nick here! +info_file = False +info_pack = True +captcha = True + +[Ev0InFetcher] +activated = False +interval = 10 +queue = False +shows = +quality = xvid +hoster = NetloadIn,RapidshareCom,MegauploadCom,HotfileCom + +[EasybytezCom] +activated = False +includeHoster = +excludeHoster = + +[XMPPInterface] +activated = False +jid = user@exmaple-jabber-server.org +pw = +tls = False +owners = me@icq-gateway.org;some@msn-gateway.org +info_file = False +info_pack = True +captcha = True + +[RehostTo] +activated = False + +[MultiHoster] +activated = True + +[MultiHome] +activated = False +interfaces = None + +[MultishareCz] +activated = False +includeHoster = +excludeHoster = rapidshare.com|uloz.to + +[HotFolder] +activated = False +folder = container +watch_file = False +keep = True +file = links.txt + +[ExternalScripts] +activated = True + +[UpdateManager] +activated = True +interval = 360 +debug = False + diff --git a/tests/config/pyload.conf.org b/tests/config/pyload.conf.org new file mode 100644 index 000000000..7fb1c8c87 --- /dev/null +++ b/tests/config/pyload.conf.org @@ -0,0 +1,75 @@ +version: 2 + +[remote] +nolocalauth = False +activated = True +port = 7227 +listenaddr = 127.0.0.1 + +[log] +log_size = 100 +log_folder = Logs +file_log = False +log_count = 5 +log_rotate = True + +[permission] +group = users +change_dl = False +change_file = False +user = user +file = 0644 +change_group = False +folder = 0755 +change_user = False + +[general] +language = en +download_folder = Downloads +checksum = False +folder_per_package = True +debug_mode = True +min_free_space = 200 +renice = 0 + +[ssl] +cert = ssl.crt +activated = False +key = ssl.key + +[webinterface] +template = default +activated = True +prefix = +server = builtin +host = 127.0.0.1 +https = False +port = 8001 + +[proxy] +username = +proxy = False +address = localhost +password = +type = http +port = 7070 + +[reconnect] +endTime = 0:00 +activated = False +method = ./reconnect.sh +startTime = 0:00 + +[download] +max_downloads = 3 +limit_speed = False +interface = +skip_existing = False +max_speed = -1 +ipv6 = False +chunks = 3 + +[downloadTime] +start = 0:00 +end = 0:00 + diff --git a/tests/config/pyload.db.org b/tests/config/pyload.db.org Binary files differnew file mode 100644 index 000000000..d340531c5 --- /dev/null +++ b/tests/config/pyload.db.org diff --git a/tests/crypterlinks.txt b/tests/crypterlinks.txt new file mode 100644 index 000000000..4ff651888 --- /dev/null +++ b/tests/crypterlinks.txt @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + +# Crypter links, append ||fail or ||#N to mark error or number of expected results (single links+packages) + +http://www.filesonic.com/folder/19906605||2 diff --git a/tests/helper/PluginTester.py b/tests/helper/PluginTester.py new file mode 100644 index 000000000..ef61385be --- /dev/null +++ b/tests/helper/PluginTester.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- + +from unittest import TestCase +from os import makedirs, remove +from os.path import exists, join, expanduser +from shutil import move +from sys import exc_clear, exc_info +from logging import log, DEBUG +from time import sleep, time +from random import randint +from glob import glob + +from pycurl import LOW_SPEED_TIME, FORM_FILE +from json import loads + +from Stubs import Thread, Core, noop + +from module.network.RequestFactory import getRequest, getURL +from module.plugins.Hoster import Hoster, Abort, Fail + +def _wait(self): + """ waits the time previously set """ + self.waiting = True + + waittime = self.pyfile.waitUntil - time() + log(DEBUG, "waiting %ss" % waittime) + + if self.wantReconnect and waittime > 300: + raise Fail("Would wait for reconnect %ss" % waittime) + elif waittime > 300: + raise Fail("Would wait %ss" % waittime) + + while self.pyfile.waitUntil > time(): + sleep(1) + if self.pyfile.abort: raise Abort + + self.waiting = False + self.pyfile.setStatus("starting") + +Hoster.wait = _wait + + +def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', + result_type='textual'): + img = self.load(url, get=get, post=post, cookies=cookies) + + id = ("%.2f" % time())[-6:].replace(".", "") + temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") + temp_file.write(img) + temp_file.close() + + Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) + + if Ocr: + log(DEBUG, "Using tesseract for captcha") + sleep(randint(3000, 5000) / 1000.0) + if self.pyfile.abort: raise Abort + + ocr = Ocr() + result = ocr.get_captcha(temp_file.name) + else: + log(DEBUG, "Using ct for captcha") + # put username and passkey into two lines in ct.conf + conf = join(expanduser("~"), "ct.conf") + if not exists(conf): raise Exception("CaptchaTrader config %s not found." % conf) + f = open(conf, "rb") + req = getRequest() + + #raise timeout threshold + req.c.setopt(LOW_SPEED_TIME, 80) + + try: + json = req.load("http://captchatrader.com/api/submit", + post={"api_key": "9f65e7f381c3af2b076ea680ae96b0b7", + "username": f.readline().strip(), + "password": f.readline().strip(), + "value": (FORM_FILE, temp_file.name), + "type": "file"}, multipart=True) + finally: + f.close() + req.close() + + response = loads(json) + log(DEBUG, str(response)) + result = response[1] + + self.cTask = response[0] + + return result + +Hoster.decryptCaptcha = decryptCaptcha + + +def respond(ticket, value): + conf = join(expanduser("~"), "ct.conf") + f = open(conf, "rb") + try: + getURL("http://captchatrader.com/api/respond", + post={"is_correct": value, + "username": f.readline().strip(), + "password": f.readline().strip(), + "ticket": ticket}) + except Exception, e : + print "CT Exception:", e + log(DEBUG, str(e)) + finally: + f.close() + + + +def invalidCaptcha(self): + log(DEBUG, "Captcha invalid") + if self.cTask: + respond(self.cTask, 0) + +Hoster.invalidCaptcha = invalidCaptcha + +def correctCaptcha(self): + log(DEBUG, "Captcha correct") + if self.cTask: + respond(self.cTask, 1) + +Hoster.correctCaptcha = correctCaptcha + +Hoster.checkForSameFiles = noop + +class PluginTester(TestCase): + @classmethod + def setUpClass(cls): + cls.core = Core() + name = "%s.%s" % (cls.__module__, cls.__name__) + for f in glob(join(name, "debug_*")): + remove(f) + + # Copy debug report to attachment dir for jenkins + @classmethod + def tearDownClass(cls): + name = "%s.%s" % (cls.__module__, cls.__name__) + if not exists(name): makedirs(name) + for f in glob("debug_*"): + move(f, join(name, f)) + + def setUp(self): + self.thread = Thread(self.core) + exc_clear() + + def tearDown(self): + exc = exc_info() + if exc != (None, None, None): + debug = self.thread.writeDebugReport() + log(DEBUG, debug) diff --git a/tests/helper/Stubs.py b/tests/helper/Stubs.py new file mode 100644 index 000000000..cfa5d6fdb --- /dev/null +++ b/tests/helper/Stubs.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- + +import sys +from os.path import abspath, dirname, join +from time import strftime + +sys.path.append(abspath(join(dirname(__file__), "..", "..", "module", "lib"))) +sys.path.append(abspath(join(dirname(__file__), "..", ".."))) + +import __builtin__ + +from module.PyPackage import PyPackage +from module.threads.BaseThread import BaseThread +from module.config.ConfigParser import ConfigParser +from module.network.RequestFactory import RequestFactory +from module.plugins.PluginManager import PluginManager +from module.common.JsEngine import JsEngine + +from logging import log, DEBUG, INFO, WARN, ERROR + + +# Do nothing +def noop(*args, **kwargs): + pass + +ConfigParser.save = noop + +class LogStub: + def debug(self, *args): + log(DEBUG, *args) + + def info(self, *args): + log(INFO, *args) + + def error(self, *args): + log(ERROR, *args) + + def warning(self, *args): + log(WARN, *args) + + +class NoLog: + def debug(self, *args): + pass + + def info(self, *args): + pass + + def error(self, *args): + log(ERROR, *args) + + def warning(self, *args): + log(WARN, *args) + + +class Core: + def __init__(self): + self.log = NoLog() + + self.api = self + self.core = self + self.debug = True + self.captcha = True + self.config = ConfigParser() + self.pluginManager = PluginManager(self) + self.requestFactory = RequestFactory(self) + __builtin__.pyreq = self.requestFactory + self.accountManager = AccountManager() + self.hookManager = self.eventManager = self.interActionManager = NoopClass() + self.js = JsEngine() + self.cache = {} + self.packageCache = {} + + self.log = LogStub() + + def getServerVersion(self): + return "TEST_RUNNER on %s" % strftime("%d %h %Y") + + def path(self, path): + return path + + def updateLink(self, *args): + pass + + def updatePackage(self, *args): + pass + + def getPackage(self, id): + return PyPackage(self, 0, "tmp", "tmp", "", "", 0, 0) + + +class NoopClass: + def __getattr__(self, item): + return noop + +class AccountManager: + + def getAccountForPlugin(self, name): + return None + +class Thread(BaseThread): + def __init__(self, core): + BaseThread.__init__(self, core) + self.plugin = None + + + def writeDebugReport(self): + if hasattr(self, "pyfile"): + dump = BaseThread.writeDebugReport(self, self.plugin.__name__, pyfile=self.pyfile) + else: + dump = BaseThread.writeDebugReport(self, self.plugin.__name__, plugin=self.plugin) + + return dump + +__builtin__._ = lambda x: x +__builtin__.pypath = abspath(join(dirname(__file__), "..", "..")) +__builtin__.hookManager = NoopClass() +__builtin__.pyreq = None
\ No newline at end of file diff --git a/tests/helper/__init__.py b/tests/helper/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/helper/__init__.py diff --git a/tests/hosterlinks.txt b/tests/hosterlinks.txt new file mode 100755 index 000000000..70dcc44f6 --- /dev/null +++ b/tests/hosterlinks.txt @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# Valid files, with md5 hash +# Please only use files around 5-15 MB and with explicit permission for redistribution + +http://download.pyload.org/random.bin +random.bin d76505d0869f9f928a17d42d66326307 +Mořská želva ( Зелёная черепаха .&+ 綠蠵龜 _@- Đồi mồi dứa ).tar 932212256dc0b0a1e71c0944eef633a4 +Mořská_želva_(_Зелёная_черепаха_.&+_綠蠵龜__@-_Đồi_mồi_dứa_).tar 932212256dc0b0a1e71c0944eef633a4 + +# Hoster links, append ||offline or ||fail to mark your expectation + +http://netload.in/datei9XirAJZs79/random.bin.htm +http://rapidshare.com/files/445996776/random.bin +http://hotfile.com/dl/101569859/2e01f04/random.bin.html +http://www.megaupload.com/?d=1JZLOP3B +http://www.shragle.com/files/f899389b/random.bin +http://www10.zippyshare.com/v/76557688/file.html +http://yourfiles.to/?d=312EC6E911 +http://depositfiles.com/files/k8la98953 +http://uploading.com/files/3896f5a1/random.bin/ + + +http://ul.to/file/o41isx||offline +http://www.4shared.com/file/-O5CBhQV/random.html||offline +http://www.4shared.com/file/-O5CBhQV/random.html||offline +http://www.fileserve.com/file/MxjZXjX||offline +http://www.share-online.biz/download.php?id=PTCOX1GL6XAH||offline +http://dl.free.fr/d4aL5dyXY||offline +http://files.mail.ru/32EW66||offline +http://www.shragle.com/files/f899389b/random.bin||offline + + +# Hoster links with fancy unicode filenames: +http://vs3iaw.1fichier.com/fr/ +http://www.4shared.com/file/rQltf2Fr/Mosk_elva___Зелная_черепаха___.html +http://bezvadata.cz/stahnout/99273_morska-zelva-.-d-i-m-i-d-a-.tar +http://www.crocko.com/A524453DA89841B4BFC4FB9125D6F186/ +http://czshare.com/2483034/zelva +http://www.easybytez.com/etvhltkg0d05 +http://www.filejungle.com/f/qX5fxT/ +http://fp.io/43798f2b/ +http://www.filesonic.com/file/yU2cU6s +http://www.fshare.vn/file/A7H8LSTP7Z/ +http://ifile.it/muwgivz +http://letitbit.net/download/67793.60a7d3745791db7271a6e6c92cfe/Mořská_želva_(_Зелёная_черепаха_.___綠蠵龜___-_Đồi_mồi_dứa_).tar.html +http://www.mediafire.com/file/n09th58z1x5r585 +http://www.quickshare.cz/stahnout-soubor/676150:morska-zelva----_-oi-moi-dua-tar_6MB +http://www.uloz.to/12553820/morska-zelva-oi-moi-dua-tar +http://www.wupload.com/file/2642593407/
\ No newline at end of file diff --git a/tests/nosetests.sh b/tests/nosetests.sh new file mode 100755 index 000000000..c68861b90 --- /dev/null +++ b/tests/nosetests.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +NS=nosetests +which nosetests2 > /dev/null && NS=nosetests2 +$NS tests/ --with-coverage --with-xunit --cover-package=module --cover-erase +coverage xml diff --git a/tests/plugin_tests.sh b/tests/plugin_tests.sh new file mode 100755 index 000000000..be06c0dc5 --- /dev/null +++ b/tests/plugin_tests.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +NS=nosetests +which nosetests2 > /dev/null && NS=nosetests2 +# must be executed within tests dir +cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +$NS HosterPluginTester.py CrypterPluginTester.py -s --with-xunit --with-coverage --cover-erase --cover-package=module.plugins --with-id +coverage xml diff --git a/tests/pyflakes.sh b/tests/pyflakes.sh new file mode 100755 index 000000000..0c7e03891 --- /dev/null +++ b/tests/pyflakes.sh @@ -0,0 +1,5 @@ +#!/bin/bash +find -name '*.py' |egrep -v '^.(/tests/|/module/lib)'|xargs pyflakes > pyflakes.log || : +# Filter warnings and strip ./ from path +cat pyflakes.log | awk -F\: '{printf "%s:%s: [E]%s\n", $1, $2, $3}' | grep -i -E -v "'_'|pypath|webinterface|pyreq|hookmanager" > pyflakes.txt +sed -i 's/^.\///g' pyflakes.txt diff --git a/tests/quit_pyload.sh b/tests/quit_pyload.sh new file mode 100755 index 000000000..e466bcb31 --- /dev/null +++ b/tests/quit_pyload.sh @@ -0,0 +1,7 @@ +#!/bin/bash +PYTHON=python +which python2 > /dev/null && PYTHON=python2 +$PYTHON pyLoadCore.py --configdir=tests/config --quit +if [ -d userplugins ]; then + rm -r userplugins +fi
\ No newline at end of file diff --git a/tests/run_pyload.sh b/tests/run_pyload.sh new file mode 100755 index 000000000..66498cd10 --- /dev/null +++ b/tests/run_pyload.sh @@ -0,0 +1,17 @@ +#/usr/bin/env bash +cp tests/config/pyload.db.org tests/config/pyload.db +cp tests/config/pyload.conf.org tests/config/pyload.conf + +PYTHON=python +which python2 > /dev/null && PYTHON=python2 + +touch pyload.out +$PYTHON pyLoadCore.py -d --configdir=tests/config > pyload.out 2> pyload.err & + +for i in {1..30}; do + grep 8001 pyload.out > /dev/null && echo "pyLoad started" && break + sleep 1 +done + +echo "pyLoad start script finished" + diff --git a/tests/sloccount.sh b/tests/sloccount.sh new file mode 100755 index 000000000..0dab4164e --- /dev/null +++ b/tests/sloccount.sh @@ -0,0 +1,2 @@ +#!/bin/bash +sloccount --duplicates --wide --details module > sloccount.sc diff --git a/tests/test_api.py b/tests/test_api.py index f8901f731..0171b46bb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,20 +1,18 @@ -# -*- coding: utf-8 -*- -from module.common import APIExerciser -from nose.tools import nottest +from unittest import TestCase +from pyLoadCore import Core +from module.common.APIExerciser import APIExerciser -class TestApi: +class TestApi(TestCase): - def __init__(self): - self.api = APIExerciser.APIExerciser(None, True, "TestUser", "pwhere") + @classmethod + def setUpClass(cls): + cls.core = Core() + cls.core.start(False, False, True) - def test_login(self): - assert self.api.api.login("crapp", "wrong pw") is False - - #takes really long, only test when needed - @nottest def test_random(self): + api = APIExerciser(self.core) - for i in range(0, 100): - self.api.testAPI() + for i in range(2000): + api.testAPI() diff --git a/tests/test_json.py b/tests/test_backends.py index ff56e8f5a..71ccedd2f 100644 --- a/tests/test_json.py +++ b/tests/test_backends.py @@ -1,14 +1,30 @@ # -*- coding: utf-8 -*- + from urllib import urlencode from urllib2 import urlopen, HTTPError from json import loads from logging import log +from module.common import APIExerciser + url = "http://localhost:8001/api/%s" -class TestJson: +class TestBackends(): + + def setUp(self): + u = urlopen(url % "login", data=urlencode({"username": "TestUser", "password": "sometestpw"})) + self.key = loads(u.read()) + assert self.key is not False + + def test_random(self): + api = APIExerciser.APIExerciser(None, True, "TestUser", "sometestpw") + + assert api.api.login("crapp", "wrong pw") is False + + for i in range(0, 1000): + api.testAPI() def call(self, name, post=None): if not post: post = {} @@ -16,11 +32,6 @@ class TestJson: u = urlopen(url % name, data=urlencode(post)) return loads(u.read()) - def setUp(self): - u = urlopen(url % "login", data=urlencode({"username": "TestUser", "password": "pwhere"})) - self.key = loads(u.read()) - assert self.key is not False - def test_wronglogin(self): u = urlopen(url % "login", data=urlencode({"username": "crap", "password": "wrongpw"})) assert loads(u.read()) is False @@ -45,4 +56,4 @@ class TestJson: except HTTPError, e: assert e.code == 404 else: - assert False
\ No newline at end of file + assert False diff --git a/tests/test_syntax.py b/tests/test_syntax.py new file mode 100644 index 000000000..a4cc53ee5 --- /dev/null +++ b/tests/test_syntax.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +from os import walk +from os.path import abspath, dirname, join + +from unittest import TestCase + +PATH = abspath(join(dirname(abspath(__file__)), "..", "")) + +# needed to register globals +from helper import Stubs + +class TestSyntax(TestCase): + pass + + +for path, dirs, files in walk(join(PATH, "module")): + + for f in files: + if not f.endswith(".py") or f.startswith("__"): continue + fpath = join(path, f) + pack = fpath.replace(PATH, "")[1:-3] #replace / and .py + imp = pack.replace("/", ".") + packages = imp.split(".") + #__import__(imp) + + # to much sideeffect when importing + if "web" in packages or "lib" in packages: continue + if "ThriftTest" in packages: continue + + # currying + def meta(imp, sig): + def _test(self=None): + __import__(imp) + + _test.func_name = sig + return _test + + # generate test methods + sig = "test_%s_%s" % (packages[-2], packages[-1]) + + + setattr(TestSyntax, sig, meta(imp, sig))
\ No newline at end of file |
