diff options
Diffstat (limited to 'module')
| -rw-r--r-- | module/Api.py | 11 | ||||
| -rw-r--r-- | module/HookManager.py | 12 | ||||
| -rw-r--r-- | module/config/ConfigParser.py | 7 | ||||
| -rw-r--r-- | module/config/converter.py | 26 | ||||
| -rw-r--r-- | module/plugins/Account.py | 3 | ||||
| -rw-r--r-- | module/plugins/Hook.py | 3 | ||||
| -rw-r--r-- | module/plugins/MultiHoster.py | 15 | ||||
| -rw-r--r-- | module/threads/BaseThread.py | 1 | ||||
| -rw-r--r-- | module/threads/DownloadThread.py | 54 | ||||
| -rw-r--r-- | module/threads/InfoThread.py | 18 | ||||
| -rw-r--r-- | module/utils/__init__.py | 37 |
11 files changed, 104 insertions, 83 deletions
diff --git a/module/Api.py b/module/Api.py index d707b18fc..11b06ff32 100644 --- a/module/Api.py +++ b/module/Api.py @@ -25,11 +25,10 @@ from itertools import chain from PyFile import PyFile -from utils import freeSpace, compare_time +from utils import freeSpace, compare_time, to_string from common.packagetools import parseNames from network.RequestFactory import getURL from remote import activated -from config.converter import to_string if activated: try: @@ -134,7 +133,7 @@ class Api(Iface): @permission(PERMS.SETTINGS) def getConfig(self): """Retrieves complete config of core. - + :return: list of `ConfigSection` """ return [ConfigSection(section, data.name, data.description, data.long_desc, [ @@ -198,7 +197,7 @@ class Api(Iface): @permission(PERMS.LIST) def statusServer(self): """Some general information about the current status of pyLoad. - + :return: `ServerStatus` """ serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), @@ -318,7 +317,7 @@ class Api(Iface): """ hoster, crypter = self.core.pluginManager.parseUrls(links) - if hoster: + if hoster: self.core.files.addLinks(hoster, pid) self.core.threadManager.createInfoThread(hoster, pid) @@ -495,7 +494,7 @@ class Api(Iface): @permission(PERMS.DELETE) def deleteFiles(self, fids): """Deletes several file entries from pyload. - + :param fids: list of file ids """ for id in fids: diff --git a/module/HookManager.py b/module/HookManager.py index 51bc706ca..b915341e3 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -26,7 +26,7 @@ from types import MethodType from module.threads.HookThread import HookThread from module.plugins.PluginManager import literal_eval -from utils import lock +from utils import lock, to_string class HookManager: """ Manages hooks, loading, unloading. """ @@ -207,17 +207,23 @@ class HookManager: if plugin.info: #copy and convert so str info[name] = dict( - [(x, str(y) if not isinstance(y, basestring) else y) for x, y in plugin.info.iteritems()]) + [(x, to_string(y)) for x, y in plugin.info.iteritems()]) return info def getInfo(self, plugin): info = {} if plugin in self.plugins and self.plugins[plugin].info: - info = dict([(x, str(y) if not isinstance(y, basestring) else y) + info = dict([(x, to_string(y)) for x, y in self.plugins[plugin].info.iteritems()]) return info + def addEventListener(self, plugin, func, event): + pass + + def addConfigHandler(self, plugin, func): + pass + def addEvent(self, *args): self.core.eventManager.addEvent(*args) diff --git a/module/config/ConfigParser.py b/module/config/ConfigParser.py index d7ecab5a0..a9e74dd20 100644 --- a/module/config/ConfigParser.py +++ b/module/config/ConfigParser.py @@ -4,15 +4,14 @@ from __future__ import with_statement from time import sleep from os.path import exists from gettext import gettext +from new_collections import namedtuple, OrderedDict +from module.utils import from_string from module.utils.fs import chmod -CONF_VERSION = 2 - -from converter import from_string -from new_collections import namedtuple, OrderedDict from default import make_config +CONF_VERSION = 2 SectionTuple = namedtuple("SectionTuple", "name description long_desc config") ConfigData = namedtuple("ConfigData", "name type description default") diff --git a/module/config/converter.py b/module/config/converter.py deleted file mode 100644 index f3b4dc327..000000000 --- a/module/config/converter.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- - -from module.utils import decode - -def to_string(value): - return str(value) if not isinstance(value, basestring) else value - -# cast value to given type, unicode for strings -def from_string(value, typ=None): - - # value is no string - if not isinstance(value, basestring): - return value - - value = decode(value) - - if typ == "int": - return int(value) - elif typ == "bool": - return True if value.lower() in ("1", "true", "on", "an", "yes") else False - elif typ == "time": - if not value: value = "0:00" - if not ":" in value: value += ":00" - return value - else: - return value
\ No newline at end of file diff --git a/module/plugins/Account.py b/module/plugins/Account.py index e5b90d95e..780a8ee69 100644 --- a/module/plugins/Account.py +++ b/module/plugins/Account.py @@ -4,8 +4,7 @@ from time import time from traceback import print_exc from threading import RLock -from module.utils import compare_time, parseFileSize, lock -from module.config.converter import from_string +from module.utils import compare_time, parseFileSize, lock, from_string from module.Api import AccountInfo from module.network.CookieJar import CookieJar diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py index c1090aa70..6e2057f03 100644 --- a/module/plugins/Hook.py +++ b/module/plugins/Hook.py @@ -156,6 +156,9 @@ class Hook(Base): def downloadFinished(self, pyfile): pass + def downloadFailed(self, pyfile): + pass + def packageFinished(self, pypack): pass diff --git a/module/plugins/MultiHoster.py b/module/plugins/MultiHoster.py index f7e560c10..047b9155e 100644 --- a/module/plugins/MultiHoster.py +++ b/module/plugins/MultiHoster.py @@ -2,8 +2,14 @@ from time import time +from module.utils import remove_chars + from Account import Account +def normalize(domain): + """ Normalize domain/plugin name, so they are comparable """ + return remove_chars(domain.strip().lower(), "-.") + #noinspection PyUnresolvedReferences class MultiHoster(Account): """ @@ -36,6 +42,15 @@ class MultiHoster(Account): """ raise NotImplementedError + + def isHosterUsuable(self, domain): + """ Determine before downloading if hoster should be used. + + :param domain: domain name + :return: True to let the MultiHoster download, False to fallback to default plugin + """ + return True + def getHosterList(self, force=False): if self.ts + self.hoster_timeout < time() or force: req = self.getAccountRequest() diff --git a/module/threads/BaseThread.py b/module/threads/BaseThread.py index f4885aadc..526913e9b 100644 --- a/module/threads/BaseThread.py +++ b/module/threads/BaseThread.py @@ -21,6 +21,7 @@ class BaseThread(Thread): Thread.__init__(self) self.setDaemon(True) self.m = manager #thread manager + self.core = manager.core self.log = manager.core.log def writeDebugReport(self, name, pyfile=None, plugin=None): diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py index 638861338..e140703d5 100644 --- a/module/threads/DownloadThread.py +++ b/module/threads/DownloadThread.py @@ -60,18 +60,18 @@ class DownloadThread(BaseThread): #this pyfile was deleted while queueing pyfile.plugin.checkForSameFiles(starting=True) - self.m.log.info(_("Download starts: %s" % pyfile.name)) + self.log.info(_("Download starts: %s" % pyfile.name)) # start download - self.m.core.hookManager.downloadPreparing(pyfile) + self.core.hookManager.downloadPreparing(pyfile) pyfile.plugin.preprocessing(self) - self.m.log.info(_("Download finished: %s") % pyfile.name) - self.m.core.hookManager.downloadFinished(pyfile) - self.m.core.files.checkPackageFinished(pyfile) + self.log.info(_("Download finished: %s") % pyfile.name) + self.core.hookManager.downloadFinished(pyfile) + self.core.files.checkPackageFinished(pyfile) except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + self.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) pyfile.setStatus("failed") pyfile.error = "Plugin does not work" self.clean(pyfile) @@ -79,7 +79,7 @@ class DownloadThread(BaseThread): except Abort: try: - self.m.log.info(_("Download aborted: %s") % pyfile.name) + self.log.info(_("Download aborted: %s") % pyfile.name) except: pass @@ -99,7 +99,7 @@ class DownloadThread(BaseThread): except Retry, e: reason = e.args[0] - self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) + self.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) self.queue.put(pyfile) continue @@ -108,16 +108,16 @@ class DownloadThread(BaseThread): if msg == "offline": pyfile.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % pyfile.name) + self.log.warning(_("Download is offline: %s") % pyfile.name) elif msg == "temp. offline": pyfile.setStatus("temp. offline") - self.m.log.warning(_("Download is temporary offline: %s") % pyfile.name) + self.log.warning(_("Download is temporary offline: %s") % pyfile.name) else: pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + self.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) pyfile.error = msg - self.m.core.hookManager.downloadFailed(pyfile) + self.core.hookManager.downloadFailed(pyfile) self.clean(pyfile) continue @@ -128,10 +128,10 @@ class DownloadThread(BaseThread): code = 0 msg = e.args - self.m.log.debug("pycurl exception %s: %s" % (code, msg)) + self.log.debug("pycurl exception %s: %s" % (code, msg)) if code in (7, 18, 28, 52, 56): - self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) + self.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) wait = time() + 60 pyfile.waitUntil = wait @@ -142,7 +142,7 @@ class DownloadThread(BaseThread): break if pyfile.abort: - self.m.log.info(_("Download aborted: %s") % pyfile.name) + self.log.info(_("Download aborted: %s") % pyfile.name) pyfile.setStatus("aborted") self.clean(pyfile) @@ -153,12 +153,12 @@ class DownloadThread(BaseThread): else: pyfile.setStatus("failed") - self.m.log.error("pycurl error %s: %s" % (code, msg)) - if self.m.core.debug: + self.log.error("pycurl error %s: %s" % (code, msg)) + if self.core.debug: print_exc() self.writeDebugReport(pyfile.pluginname, pyfile) - self.m.core.hookManager.downloadFailed(pyfile) + self.core.hookManager.downloadFailed(pyfile) self.clean(pyfile) continue @@ -166,34 +166,34 @@ class DownloadThread(BaseThread): except SkipDownload, e: pyfile.setStatus("skipped") - self.m.log.info( - _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) + self.log.info(_("Download skipped: %(name)s due to %(plugin)s") + % {"name": pyfile.name, "plugin": e.message}) self.clean(pyfile) - self.m.core.files.checkPackageFinished(pyfile) + self.core.files.checkPackageFinished(pyfile) self.active = False - self.m.core.files.save() + self.core.files.save() continue except Exception, e: pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + self.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) pyfile.error = str(e) - if self.m.core.debug: + if self.core.debug: print_exc() self.writeDebugReport(pyfile.pluginname, pyfile) - self.m.core.hookManager.downloadFailed(pyfile) + self.core.hookManager.downloadFailed(pyfile) self.clean(pyfile) continue finally: - self.m.core.files.save() + self.core.files.save() pyfile.checkIfProcessed() exc_clear() @@ -202,7 +202,7 @@ class DownloadThread(BaseThread): self.active = False pyfile.finishIfDone() - self.m.core.files.save() + self.core.files.save() def put(self, job): diff --git a/module/threads/InfoThread.py b/module/threads/InfoThread.py index c1e4458ef..5f21d487c 100644 --- a/module/threads/InfoThread.py +++ b/module/threads/InfoThread.py @@ -5,9 +5,8 @@ from time import time from traceback import print_exc from module.Api import OnlineStatus -from module.PyFile import PyFile from module.common.packagetools import parseNames -from module.utils import has_method +from module.utils import has_method, accumulate from BaseThread import BaseThread @@ -29,16 +28,9 @@ class InfoThread(BaseThread): def run(self): """run method""" - plugins = {} + plugins = accumulate(self.data) crypter = {} - for url, plugin in self.data: - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] - - # filter out crypter plugins for name in self.m.core.pluginManager.getPlugins("crypter"): if name in plugins: @@ -68,11 +60,7 @@ class InfoThread(BaseThread): self.m.log.error("Could not decrypt container.") data = [] - for url, plugin in data: - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] + accumulate(data, plugins) self.m.infoResults[self.rid] = {} diff --git a/module/utils/__init__.py b/module/utils/__init__.py index 46621c685..8457eba07 100644 --- a/module/utils/__init__.py +++ b/module/utils/__init__.py @@ -150,6 +150,43 @@ def has_method(obj, name): """ checks if 'name' was defined in obj, (false if it was inhereted) """ return name in obj.__dict__ +def accumulate(it, inv_map=None): + """ accumulate (key, value) data to {value : [keylist]} dictionary """ + if not inv_map: + inv_map = {} + + for key, value in it: + if value in inv_map: + inv_map[value].append(key) + else: + inv_map[value] = [key] + + return inv_map + +def to_string(value): + return str(value) if not isinstance(value, basestring) else value + +def from_string(value, typ=None): + """ cast value to given type, unicode for strings """ + + # value is no string + if not isinstance(value, basestring): + return value + + value = decode(value) + + if typ == "int": + return int(value) + elif typ == "bool": + return True if value.lower() in ("1", "true", "on", "an", "yes") else False + elif typ == "time": + if not value: value = "0:00" + if not ":" in value: value += ":00" + return value + else: + return value + + def html_unescape(text): """Removes HTML or XML character references and entities from a text string""" return re.sub("&#?\w+;", fixup, text) |