diff options
| -rw-r--r-- | module/Api.py | 30 | ||||
| -rw-r--r-- | module/HookManager.py | 8 | ||||
| -rw-r--r-- | module/PluginThread.py | 66 | ||||
| -rw-r--r-- | module/PyFile.py | 4 | ||||
| -rw-r--r-- | module/ThreadManager.py | 4 | ||||
| -rw-r--r-- | module/remote/thriftbackend/pyload.thrift | 1 | ||||
| -rwxr-xr-x | module/remote/thriftbackend/thriftgen/pyload/Pyload-remote | 7 | ||||
| -rw-r--r-- | module/remote/thriftbackend/thriftgen/pyload/Pyload.py | 100 | ||||
| -rwxr-xr-x | pyLoadCli.py | 39 |
9 files changed, 223 insertions, 36 deletions
diff --git a/module/Api.py b/module/Api.py index 221809dc2..0cc2dc1d4 100644 --- a/module/Api.py +++ b/module/Api.py @@ -28,8 +28,6 @@ from PyFile import PyFile from database.UserDatabase import ROLE from utils import freeSpace, compare_time from common.packagetools import parseNames -from network.RequestFactory import getURL - class Api(Iface): """ @@ -266,8 +264,6 @@ class Api(Iface): return pid def parseURLs(self, html): - html = getURL(html) - # TODO parse urls = [] @@ -286,7 +282,7 @@ class Api(Iface): return plugins - def checkOnlineStatus(self, urls): + def checkOnlineStatus(self, urls, container=None): """ initiates online status check :param urls: @@ -294,7 +290,7 @@ class Api(Iface): """ data = self.core.pluginManager.parseUrls(urls) - rid = self.core.threadManager.createResultThread(data) + rid = self.core.threadManager.createResultThread(data, False, container) tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data] data = parseNames(tmp) @@ -307,6 +303,20 @@ class Api(Iface): return OnlineCheck(rid, result) + def checkOnlineStatusContainer(self, urls, container, data): + """ checks online status of files and container file + + :param urls: list of urls + :param container: container file name + :param data: file content + :return: online check + """ + th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb") + th.write(str(data)) + th.close() + + return self.checkOnlineStatus(urls, th.name) + def pollResults(self, rid): """ Polls the result available for ResultID @@ -351,7 +361,7 @@ class Api(Iface): :param dest: `Destination` :return: None """ - data = self.core.pluginManager.parseUrls(urls) + data = self.core.pluginManager.parseUrls(links) self.core.threadManager.createResultThread(data, True) @@ -846,11 +856,7 @@ class Api(Iface): :param plugin: pluginname :return: dict of attr names mapped to value {"name": value} """ - info = self.core.hookManager.getAllInfo() - if plugin in info: - return info[plugin] - else: - return {} + return self.core.hookManager.getInfo(plugin) def changePassword(self, user, oldpw, newpw): """ changes password for specific user """ diff --git a/module/HookManager.py b/module/HookManager.py index c67acd34a..5876debe4 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -272,6 +272,14 @@ class HookManager: return info + def getInfo(self, plugin): + info = {} + if plugin in self.pluginMap and self.pluginMap[plugin].info: + info = dict([(x, str(y) if type(y) != basestring else y) + for x, y in self.pluginMap[plugin].info.iteritems()]) + + return info + def addEvent(self, event, func): """Adds an event listener for event name""" if event in self.events: diff --git a/module/PluginThread.py b/module/PluginThread.py index 0a8151a25..b32361f4e 100644 --- a/module/PluginThread.py +++ b/module/PluginThread.py @@ -407,7 +407,7 @@ class HookThread(PluginThread): class InfoThread(PluginThread): - def __init__(self, manager, data, pid=-1, rid=-1, add=False): + def __init__(self, manager, data, pid=-1, rid=-1, add=False, container=None): """Constructor""" PluginThread.__init__(self, manager) @@ -417,6 +417,7 @@ class InfoThread(PluginThread): self.rid = rid #result id self.add = add #add packages instead of return result + self.container = container #container file self.cache = [] #accumulated data @@ -453,10 +454,9 @@ class InfoThread(PluginThread): self.updateCache(pluginname, result) + packs = parseNames([(name, url) for name, x, y, url in self.cache]) - packs = parseNames([(name, url) for name, x,y, url in self.cache]) - - self.m.core.log.debug("Fetched and generated %d packages" % len(packs)) + self.m.log.debug("Fetched and generated %d packages" % len(packs)) for k, v in packs: self.m.core.api.addPackage(k, v) @@ -466,6 +466,22 @@ class InfoThread(PluginThread): else: #post the results + + if self.container: + #attach container content + try: + data = self.decryptContainer() + except: + print_exc() + self.m.log.error("Could not decrypt container.") + data = [] + + for url, plugin in data: + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + self.m.infoResults[self.rid] = {} for pluginname, urls in plugins.iteritems(): @@ -527,11 +543,11 @@ class InfoThread(PluginThread): process.append(url) if result: - self.m.core.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) + self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) cb(pluginname, result) if process: - self.m.core.log.debug("Run Info Fetching for %s" % pluginname) + self.m.log.debug("Run Info Fetching for %s" % pluginname) for result in plugin.getInfo(process): #result = [ .. (name, size, status, url) .. ] if not type(result) == list: result = [result] @@ -541,13 +557,41 @@ class InfoThread(PluginThread): cb(pluginname, result) - self.m.core.log.debug("Finished Info Fetching for %s" % pluginname) + self.m.log.debug("Finished Info Fetching for %s" % pluginname) except Exception, e: - self.m.core.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % - {"name": pluginname, "err": str(e)}) + self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % + {"name": pluginname, "err": str(e)}) if self.m.core.debug: print_exc() - #TODO: generate default results + # generate default results if err: - pass + result = [(url, 0, 3, url) for url in urls] + cb(pluginname, result) + + + def decryptContainer(self): + url, plugin = self.m.core.pluginManager.parseUrls([self.container])[0] + # decrypt only container + if plugin in self.m.core.pluginManager.containerPlugins: + + # dummy pyfile + pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) + + pyfile.initPlugin() + + # little plugin lifecycle + pyfile.plugin.setup() + pyfile.plugin.loadToDisk() + pyfile.plugin.decrypt(pyfile) + pyfile.plugin.deleteTmp() + + for pack in pyfile.plugin.packages: + pyfile.plugin.urls.extend(pack[1]) + + data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) + pyfile.release() + + return data + + return [] diff --git a/module/PyFile.py b/module/PyFile.py index 540933a93..9390c9cca 100644 --- a/module/PyFile.py +++ b/module/PyFile.py @@ -132,7 +132,9 @@ class PyFile(object): @lock def release(self): """sync and remove from cache""" - self.sync() + # file has valid package + if self.packageid > 0: + self.sync() if hasattr(self, "plugin") and self.plugin: self.plugin.clean() diff --git a/module/ThreadManager.py b/module/ThreadManager.py index ce5b7bd1d..159b495cd 100644 --- a/module/ThreadManager.py +++ b/module/ThreadManager.py @@ -88,14 +88,14 @@ class ThreadManager: PluginThread.InfoThread(self, data, pid) @lock - def createResultThread(self, data, add=False): + def createResultThread(self, data, add=False, container=None): """ creates a thread to fetch online status, returns result id """ self.timestamp = time() + 5 * 60 rid = self.resultIDs self.resultIDs += 1 - PluginThread.InfoThread(self, data, rid=rid, add=add) + PluginThread.InfoThread(self, data, rid=rid, add=add, container=container) return rid diff --git a/module/remote/thriftbackend/pyload.thrift b/module/remote/thriftbackend/pyload.thrift index d73021725..ebe1890e3 100644 --- a/module/remote/thriftbackend/pyload.thrift +++ b/module/remote/thriftbackend/pyload.thrift @@ -221,6 +221,7 @@ service Pyload { // parses results and generates packages OnlineCheck checkOnlineStatus(1: LinkList urls), + OnlineCheck checkOnlineStatusContainer(1: LinkList urls, 2: string filename, 3: binary data) // poll results from previosly started online check OnlineCheck pollResults(1: ResultID rid), diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote index cdae7ac2a..8c2b368e1 100755 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote @@ -43,6 +43,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print ' checkURLs(LinkList urls)' print ' parseURLs(string html)' print ' OnlineCheck checkOnlineStatus(LinkList urls)' + print ' OnlineCheck checkOnlineStatusContainer(LinkList urls, string filename, string data)' print ' OnlineCheck pollResults(ResultID rid)' print ' statusDownloads()' print ' PackageData getPackageData(PackageID pid)' @@ -261,6 +262,12 @@ elif cmd == 'checkOnlineStatus': sys.exit(1) pp.pprint(client.checkOnlineStatus(eval(args[0]),)) +elif cmd == 'checkOnlineStatusContainer': + if len(args) != 3: + print 'checkOnlineStatusContainer requires 3 args' + sys.exit(1) + pp.pprint(client.checkOnlineStatusContainer(eval(args[0]),args[1],args[2],)) + elif cmd == 'pollResults': if len(args) != 1: print 'pollResults requires 1 args' diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py index 008c42a56..816c6e7d1 100644 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py @@ -106,6 +106,15 @@ class Iface(object): """ pass + def checkOnlineStatusContainer(self, urls, filename, data): + """ + Parameters: + - urls + - filename + - data + """ + pass + def pollResults(self, rid): """ Parameters: @@ -956,6 +965,40 @@ class Client(Iface): return result.success raise TApplicationException(TApplicationException.MISSING_RESULT, "checkOnlineStatus failed: unknown result"); + def checkOnlineStatusContainer(self, urls, filename, data): + """ + Parameters: + - urls + - filename + - data + """ + self.send_checkOnlineStatusContainer(urls, filename, data) + return self.recv_checkOnlineStatusContainer() + + def send_checkOnlineStatusContainer(self, urls, filename, data): + self._oprot.writeMessageBegin('checkOnlineStatusContainer', TMessageType.CALL, self._seqid) + args = checkOnlineStatusContainer_args() + args.urls = urls + args.filename = filename + args.data = data + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_checkOnlineStatusContainer(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = checkOnlineStatusContainer_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "checkOnlineStatusContainer failed: unknown result"); + def pollResults(self, rid): """ Parameters: @@ -2367,6 +2410,7 @@ class Processor(Iface, TProcessor): self._processMap["checkURLs"] = Processor.process_checkURLs self._processMap["parseURLs"] = Processor.process_parseURLs self._processMap["checkOnlineStatus"] = Processor.process_checkOnlineStatus + self._processMap["checkOnlineStatusContainer"] = Processor.process_checkOnlineStatusContainer self._processMap["pollResults"] = Processor.process_pollResults self._processMap["statusDownloads"] = Processor.process_statusDownloads self._processMap["getPackageData"] = Processor.process_getPackageData @@ -2651,6 +2695,17 @@ class Processor(Iface, TProcessor): oprot.writeMessageEnd() oprot.trans.flush() + def process_checkOnlineStatusContainer(self, seqid, iprot, oprot): + args = checkOnlineStatusContainer_args() + args.read(iprot) + iprot.readMessageEnd() + result = checkOnlineStatusContainer_result() + result.success = self._handler.checkOnlineStatusContainer(args.urls, args.filename, args.data) + oprot.writeMessageBegin("checkOnlineStatusContainer", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_pollResults(self, seqid, iprot, oprot): args = pollResults_args() args.read(iprot) @@ -3784,6 +3839,51 @@ class checkOnlineStatus_result(TBase): self.success = success +class checkOnlineStatusContainer_args(TBase): + """ + Attributes: + - urls + - filename + - data + """ + + __slots__ = [ + 'urls', + 'filename', + 'data', + ] + + thrift_spec = ( + None, # 0 + (1, TType.LIST, 'urls', (TType.STRING,None), None, ), # 1 + (2, TType.STRING, 'filename', None, None, ), # 2 + (3, TType.STRING, 'data', None, None, ), # 3 + ) + + def __init__(self, urls=None, filename=None, data=None,): + self.urls = urls + self.filename = filename + self.data = data + + +class checkOnlineStatusContainer_result(TBase): + """ + Attributes: + - success + """ + + __slots__ = [ + 'success', + ] + + thrift_spec = ( + (0, TType.STRUCT, 'success', (OnlineCheck, OnlineCheck.thrift_spec), None, ), # 0 + ) + + def __init__(self, success=None,): + self.success = success + + class pollResults_args(TBase): """ Attributes: diff --git a/pyLoadCli.py b/pyLoadCli.py index 9760c08b7..9c456d422 100755 --- a/pyLoadCli.py +++ b/pyLoadCli.py @@ -23,7 +23,7 @@ from getopt import GetoptError, getopt import gettext import os from os import _exit -from os.path import join, exists, abspath +from os.path import join, exists, abspath, basename import sys from sys import exit from threading import Thread, Lock @@ -321,23 +321,28 @@ class Cli: if exists(join(OWD, url)): f = open(join(OWD, url), "rb") links.extend([x.strip() for x in f.readlines() if x.strip()]) + f.close() else: links.append(url) print _("Checking %d links:") % len(links) print rid = client.checkOnlineStatus(links).rid - while True: - sleep(1) - result = client.pollResults(rid) - for url, status in result.data.iteritems(): - if status.status == 2: check = "Online" - elif status.status == 1: check = "Offline" - else: check = "Unknown" + self.printOnlineCheck(client, rid) - print "%-30s: %-30s %-8s\t %s" % (url, status.name, formatSize(status.size), check) - if result.rid == -1: break + elif command == "check_container": + path = args[0] + if not exists(join(OWD, path)): + print _("File does not exists.") + return + + f = open(join(OWD, path), "rb") + content = f.read() + f.close() + + rid = client.checkOnlineStatusContainer([], basename(f.name), content).rid + self.printOnlineCheck(client, rid) elif command == "pause": @@ -363,6 +368,19 @@ class Cli: else: print_commands() + def printOnlineCheck(self, client, rid): + while True: + sleep(1) + result = client.pollResults(rid) + for url, status in result.data.iteritems(): + if status.status == 2: check = "Online" + elif status.status == 1: check = "Offline" + else: check = "Unknown" + + print "%-30s: %-30s %-8s\t %s" % (url, status.name, formatSize(status.size), check) + + if result.rid == -1: break + class RefreshThread(Thread): def __init__(self, cli): @@ -446,6 +464,7 @@ def print_commands(): ("restart_file <fid> <fid2>...", _("Restart files")), ("restart_package <pid> <pid2>...", _("Restart packages")), ("check <linklist|url> ...", _("Check online status")), + ("check_container path", _("Checks online status of a container file")), ("pause", _("Pause the server")), ("unpause", _("continue downloads")), ("toggle", _("Toggle pause/unpause")), |