diff options
Diffstat (limited to 'module')
| -rw-r--r-- | module/Api.py | 53 | ||||
| -rw-r--r-- | module/PluginThread.py | 170 | ||||
| -rw-r--r-- | module/ThreadManager.py | 35 | ||||
| -rw-r--r-- | module/common/packagetools.py | 147 | ||||
| -rw-r--r-- | module/plugins/hoster/BasePlugin.py | 5 | ||||
| -rw-r--r-- | module/remote/thriftbackend/pyload.thrift | 13 | ||||
| -rwxr-xr-x | module/remote/thriftbackend/thriftgen/pyload/Pyload-remote | 14 | ||||
| -rw-r--r-- | module/remote/thriftbackend/thriftgen/pyload/Pyload.py | 181 | ||||
| -rw-r--r-- | module/remote/thriftbackend/thriftgen/pyload/ttypes.py | 12 |
9 files changed, 475 insertions, 155 deletions
diff --git a/module/Api.py b/module/Api.py index fd65dff92..dbdd2b04c 100644 --- a/module/Api.py +++ b/module/Api.py @@ -27,6 +27,7 @@ from remote.thriftbackend.thriftgen.pyload.Pyload import Iface from module.PyFile import PyFile from module.database.UserDatabase import ROLE from module.utils import freeSpace, compare_time +from module.common.packagetools import parseNames class Api(Iface): @@ -201,16 +202,6 @@ class Api(Iface): except: return ['No log available'] - def parseURLs(self, html): - #TODO implement - pass - - def checkURLs(self, urls): - pass - - def checkOnlineStatus(self, urls): - pass - def isTimeDownload(self): """Checks if pyload will start new downloads according to time in config . @@ -273,6 +264,48 @@ class Api(Iface): return pid + def parseURLs(self, html): + #TODO implement + pass + + def checkURLs(self, urls): + data = self.core.pluginManager.parseUrls(urls) + plugins = {} + + for url, plugin in data: + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + return plugins + + def checkOnlineStatus(self, urls): + data = self.core.pluginManager.parseUrls(urls) + return self.core.threadManager.createResultThread(data) + + def pollResults(self, rid): + pass + + def generatePackages(self, links): + """ Parses links, generates packages names only from urls + + :param links: list of urls + :return: package names mapt to urls + """ + result = parseNames((x,x) for x in links) + return result + + def generateAndAddPackages(self, links, dest=Destination.Queue): + """Generates and add packages + + :param links: list of urls + :param dest: `Destination` + :return: list of package ids + """ + return [self.addPackage(name, urls, dest) for name, urls + in self.generatePackages(links).iteritems()] + def getPackageData(self, pid): """Returns complete information about package, and included files. diff --git a/module/PluginThread.py b/module/PluginThread.py index e0e3b17c9..a44981c52 100644 --- a/module/PluginThread.py +++ b/module/PluginThread.py @@ -30,8 +30,10 @@ from types import MethodType from pycurl import error -from module.PyFile import PyFile -from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload +from PyFile import PyFile +from plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload +from common.packagetools import parseNames +from remote.thriftbackend.thriftgen.pyload.ttypes import OnlineStatus class PluginThread(Thread): @@ -46,7 +48,8 @@ class PluginThread(Thread): def writeDebugReport(self, pyfile): - dump = "pyLoad %s Debug Report of %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % (self.m.core.api.getServerVersion(), pyfile.pluginname, format_exc()) + dump = "pyLoad %s Debug Report of %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( + self.m.core.api.getServerVersion(), pyfile.pluginname, format_exc()) tb = exc_info()[2] stack = [] @@ -55,18 +58,17 @@ class PluginThread(Thread): tb = tb.tb_next for frame in stack[1:]: - dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, - frame.f_code.co_filename, - frame.f_lineno) + frame.f_code.co_filename, + frame.f_lineno) for key, value in frame.f_locals.items(): dump += "\t%20s = " % key try: dump += pformat(value) + "\n" except Exception, e: - dump += "<ERROR WHILE PRINTING VALUE> "+ str(e) +"\n" - + dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" + del frame del stack #delete it just to be sure... @@ -80,7 +82,7 @@ class PluginThread(Thread): try: dump += pformat(attr) + "\n" except Exception, e: - dump += "<ERROR WHILE PRINTING VALUE> "+ str(e) +"\n" + dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" dump += "\nPYFILE OBJECT DUMP: \n\n" @@ -91,14 +93,11 @@ class PluginThread(Thread): try: dump += pformat(attr) + "\n" except Exception, e: - dump += "<ERROR WHILE PRINTING VALUE> "+ str(e) +"\n" - + dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" if pyfile.pluginname in self.m.core.config.plugin: dump += "\n\nCONFIG: \n\n" - dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) +"\n" - - + dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" dump_name = "debug_%s_%s.txt" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) self.m.core.log.info("Debug Report written to %s" % dump_name) @@ -142,7 +141,6 @@ class DownloadThread(PluginThread): return True try: - if not pyfile.hasPlugin(): continue #this pyfile was deleted while queueing @@ -154,7 +152,6 @@ class DownloadThread(PluginThread): pyfile.plugin.preprocessing(self) except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) pyfile.setStatus("failed") pyfile.error = "Plugin does not work" @@ -164,9 +161,9 @@ class DownloadThread(PluginThread): except Abort: try: self.m.log.info(_("Download aborted: %s") % pyfile.name) - except : + except: pass - + pyfile.setStatus("aborted") self.clean(pyfile) @@ -182,14 +179,12 @@ class DownloadThread(PluginThread): continue except Retry, e: - reason = e.args[0] - self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name" : pyfile.name, "msg": reason}) + self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) self.queue.put(pyfile) continue except Fail, e: - msg = e.args[0] if msg == "offline": @@ -218,7 +213,7 @@ class DownloadThread(PluginThread): if code in (7, 18, 28, 52, 56): self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) wait = time() + 60 - + pyfile.waitUntil = wait pyfile.setStatus("waiting") while time() < wait: @@ -247,10 +242,10 @@ class DownloadThread(PluginThread): continue except SkipDownload, e: - pyfile.setStatus("skipped") - self.m.log.info(_("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) + self.m.log.info( + _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) self.clean(pyfile) @@ -279,7 +274,6 @@ class DownloadThread(PluginThread): self.m.core.files.checkAllLinksProcessed() exc_clear() - self.m.log.info(_("Download finished: %s") % pyfile.name) #pyfile.plugin.req.clean() @@ -302,7 +296,6 @@ class DownloadThread(PluginThread): self.put("quit") - class DecrypterThread(PluginThread): """thread for decrypting""" @@ -330,12 +323,10 @@ class DecrypterThread(PluginThread): self.active.plugin.preprocessing(self) except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) return except Fail, e: - msg = e.args[0] if msg == "offline": @@ -343,28 +334,25 @@ class DecrypterThread(PluginThread): self.m.log.warning(_("Download is offline: %s") % self.active.name) else: self.active.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % { "name" : self.active.name, "msg":msg }) + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": msg}) self.active.error = msg return - + except Abort: - self.m.log.info(_("Download aborted: %s") % pyfile.name) pyfile.setStatus("aborted") - + return except Retry: - self.m.log.info(_("Retrying %s") % self.active.name) retry = True return self.run() except Exception, e: - self.active.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % { "name" : self.active.name, "msg" :str(e) }) + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": str(e)}) self.active.error = str(e) if self.m.core.debug: @@ -391,6 +379,7 @@ class DecrypterThread(PluginThread): if not retry: pyfile.delete() + class HookThread(PluginThread): """thread for hooks""" @@ -403,7 +392,7 @@ class HookThread(PluginThread): self.active = pyfile m.localThreads.append(self) - + if isinstance(pyfile, PyFile): pyfile.setStatus("processing") @@ -411,22 +400,26 @@ class HookThread(PluginThread): def run(self): self.f(self.active) - + self.m.localThreads.remove(self) if isinstance(self.active, PyFile): self.active.finishIfDone() class InfoThread(PluginThread): - #---------------------------------------------------------------------- - def __init__(self, manager, data, pid): + def __init__(self, manager, data, pid=-1, rid=-1): """Constructor""" PluginThread.__init__(self, manager) self.data = data self.pid = pid # package id # [ .. (name, plugin) .. ] + + self.rid = rid #result id + + self.cache = [] #accumulated data + self.start() #---------------------------------------------------------------------- @@ -441,21 +434,88 @@ class InfoThread(PluginThread): else: plugins[plugin] = [url] - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): - try: - self.m.core.log.debug("Run Info Fetching for %s" % pluginname) - for result in plugin.getInfo(urls): - #result = [ .. (name, size, status, url) .. ] - if not type(result) == list: result = [result] - self.m.core.files.updateFileInfo(result, self.pid) + #directly write to database + if self.pid > -1: + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) + self.m.core.files.save() - self.m.core.log.debug("Finished Info Fetching for %s" % pluginname) + else: #post the results - self.m.core.files.save() - except Exception, e: - self.m.core.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % {"name": pluginname, "err": str(e)} ) - if self.m.core.debug: - print_exc() - + self.m.infoResults[self.rid] = {} + + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) + + #force to process cache + if self.cache: + self.updateResult(pluginname, [], True) + + else: + #generate default result + pass + + self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = [] + + + def updateDB(self, plugin, result): + self.m.core.files.updateFileInfo(result, self.pid) + + def updateResult(self, plugin, result, force=False): + #parse package name and generate result + #accumulate results + + self.cache.extend(result) + + if len(self.cache) > 20 or force: + #used for package generating + tmp = [(name, (url, OnlineStatus(name, plugin, status, int(size)))) + for name, size, status, url in self.cache] + + result = parseNames(tmp) + for k in result.iterkeys(): + result[k] = dict(result[k]) + + print result + + self.cache = [] + + def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): + try: + result = [] #result loaded from cache + process = [] #urls to process + for url in urls: + if url in self.m.infoCache: + result.append(self.m.infoCache[url]) + else: + process.append(url) + + if result: + self.m.core.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) + cb(pluginname, result) + + if process: + self.m.core.log.debug("Run Info Fetching for %s" % pluginname) + for result in plugin.getInfo(process): + #result = [ .. (name, size, status, url) .. ] + if not type(result) == list: result = [result] + + for res in result: + self.m.infoCache[res[3]] = res + + cb(pluginname, result) + + self.m.core.log.debug("Finished Info Fetching for %s" % pluginname) + except Exception, e: + self.m.core.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % + {"name": pluginname, "err": str(e)}) + if self.m.core.debug: + print_exc() + + #TODO: generate default results + if err: + pass diff --git a/module/ThreadManager.py b/module/ThreadManager.py index 2c3d9af95..0ee59b427 100644 --- a/module/ThreadManager.py +++ b/module/ThreadManager.py @@ -33,11 +33,11 @@ from module.PyFile import PyFile from module.network.RequestFactory import getURL from module.utils import freeSpace -######################################################################## + class ThreadManager: """manages the download threads, assign jobs, reconnect etc""" - #---------------------------------------------------------------------- + def __init__(self, core): """Constructor""" self.core = core @@ -52,21 +52,31 @@ class ThreadManager: self.reconnecting.clear() self.downloaded = 0 #number of files downloaded since last cleanup + # some operations require to fetch url info from hoster, so we caching them so it wont be done twice + # contains a timestamp and will be purged after timeout + self.infoCache = {} + + # pool of ids for online check + self.resultIDs = 0 + + # threads which are fetching hoster results + self.infoResults = {} + #timeout for cache purge + self.timestamp = 0 + pycurl.global_init(pycurl.GLOBAL_DEFAULT) for i in range(0, self.core.config.get("download", "max_downloads")): self.createThread() - - #---------------------------------------------------------------------- def createThread(self): """create a download thread""" thread = PluginThread.DownloadThread(self) self.threads.append(thread) - #---------------------------------------------------------------------- + def createInfoThread(self, data, pid): """ start a thread whichs fetches online status and other infos @@ -75,19 +85,28 @@ class ThreadManager: PluginThread.InfoThread(self, data, pid) + def createResultThread(self, data): + """ creates a thread to fetch online status, returns result id """ + + rid = self.resultIDs + self.resultIDs += 1 + + PluginThread.InfoThread(self, data, rid=rid) + + return rid + - #---------------------------------------------------------------------- def downloadingIds(self): """get a list of the currently downloading pyfile's ids""" return [x.active.id for x in self.threads if x.active and isinstance(x.active, PyFile)] - #---------------------------------------------------------------------- + def processingIds(self): """get a id list of all pyfiles processed""" return [x.active.id for x in self.threads + self.localThreads if x.active and isinstance(x.active, PyFile)] - #---------------------------------------------------------------------- + def work(self): """run all task which have to be done (this is for repetivive call by core)""" try: diff --git a/module/common/packagetools.py b/module/common/packagetools.py index 175c48937..4682b0dc1 100644 --- a/module/common/packagetools.py +++ b/module/common/packagetools.py @@ -5,98 +5,86 @@ import re from urlparse import urlparse +def matchFirst(string, *args): + """ matches against list of regexp and returns first match""" + for patternlist in args: + for pattern in patternlist: + r = pattern.search(string) + if r is not None: + name = r.group(1) + return name + + return string + + def parseNames(files): + """ Generates packages names from name, data lists + + :param files: list of (name, data) + :return: packagenames mapt to data lists (eg. urls) + """ packs = {} endings = "\\.(3gp|7zip|7z|abr|ac3|aiff|aifc|aif|ai|au|avi|bin|bz2|cbr|cbz|ccf|cue|cvd|chm|dta|deb|divx|djvu|dlc|dmg|doc|docx|dot|eps|exe|ff|flv|f4v|gsd|gif|gz|iwd|iso|ipsw|java|jar|jpg|jpeg|jdeatme|load|mws|mw|m4v|m4a|mkv|mp2|mp3|mp4|mov|movie|mpeg|mpe|mpg|msi|msu|msp|nfo|npk|oga|ogg|ogv|otrkey|pkg|png|pdf|pptx|ppt|pps|ppz|pot|psd|qt|rmvb|rm|rar|ram|ra|rev|rnd|r\\d+|rpm|run|rsdf|rtf|sh(!?tml)|srt|snd|sfv|swf|tar|tif|tiff|ts|txt|viv|vivo|vob|wav|wmv|xla|xls|xpi|zeno|zip|z\\d+|_[_a-z]{2}|\\d+$)" - pat0 = re.compile("(.*)(\\.|_|-)pa?r?t?\\.?[0-9]+.(rar|exe)$", re.I) - pat1 = re.compile("(.*)(\\.|_|-)part\\.?[0]*[1].(rar|exe)$", re.I) - pat3 = re.compile("(.*)\\.rar$", re.I) - pat4 = re.compile("(.*)\\.r\\d+$", re.I) - pat5 = re.compile("(.*)(\\.|_|-)\\d+$", re.I) - rarPats = [ pat0, pat1, pat3, pat4, pat5 ] + rarPats = [re.compile("(.*)(\\.|_|-)pa?r?t?\\.?[0-9]+.(rar|exe)$", re.I), + re.compile("(.*)(\\.|_|-)part\\.?[0]*[1].(rar|exe)$", re.I), + re.compile("(.*)\\.rar$", re.I), + re.compile("(.*)\\.r\\d+$", re.I), + re.compile("(.*)(\\.|_|-)\\d+$", re.I)] - pat6 = re.compile("(.*)\\.zip$", re.I) - pat7 = re.compile("(.*)\\.z\\d+$", re.I) - pat8 = re.compile("(?is).*\\.7z\\.[\\d]+$", re.I) - pat9 = re.compile("(.*)\\.a.$", re.I) - zipPats = [ pat6, pat7, pat8, pat9 ] + zipPats = [re.compile("(.*)\\.zip$", re.I), + re.compile("(.*)\\.z\\d+$", re.I), + re.compile("(?is).*\\.7z\\.[\\d]+$", re.I), + re.compile("(.*)\\.a.$", re.I)] - pat10 = re.compile("(.*)\\._((_[a-z])|([a-z]{2}))(\\.|$)") - pat11 = re.compile("(.*)(\\.|_|-)[\\d]+(" + endings + "$)", re.I) - ffsjPats = [ pat10, pat11 ] + ffsjPats = [re.compile("(.*)\\._((_[a-z])|([a-z]{2}))(\\.|$)"), + re.compile("(.*)(\\.|_|-)[\\d]+(" + endings + "$)", re.I)] - pat12 = re.compile("(\\.?CD\\d+)", re.I) - pat13 = re.compile("(\\.?part\\d+)", re.I) + iszPats = [re.compile("(.*)\\.isz$", re.I), + re.compile("(.*)\\.i\\d{2}$", re.I)] - pat14 = re.compile("(.+)[\\.\\-_]+$") + pat1 = re.compile("(\\.?CD\\d+)", re.I) + pat2 = re.compile("(\\.?part\\d+)", re.I) - pat17 = re.compile("(.+)\\.\\d+\\.xtm$") + pat3 = re.compile("(.+)[\\.\\-_]+$") + pat4 = re.compile("(.+)\\.\\d+\\.xtm$") - pat18 = re.compile("(.*)\\.isz$", re.I) - pat19 = re.compile("(.*)\\.i\\d{2}$", re.I) - iszPats = [ pat18, pat19 ] - for file in files: + for file, url in files: # remove trailing / name = file.rstrip('/') + # extract last path part .. if there is a path split = name.rsplit("/", 1) if len(split) > 1: name = split.pop(1) + #check if a already existing package may be ok for this file + # found = False + # for pack in packs: + # if pack in file: + # packs[pack].append(url) + # found = True + # break + # + # if found: continue - #check if a already existing package may be ok for this file - found = False - for name in packs: - if name in file: - packs[name].append(file) - found = True - break - - if found: continue - - - # unrar pattern - for pattern in rarPats: - r = pattern.search(name) - if r is not None: - name = r.group(1) - break - - # 7zip/zip and hjmerge pattern - for pattern in zipPats: - r = pattern.search(name) - if r is not None: - name = r.group(1) - break - - # isz pattern - for pattern in iszPats: - r = pattern.search(name) - if r is not None: - name = r.group(1) - break + # unrar pattern, 7zip/zip and hjmerge pattern, isz pattern, FFSJ pattern + name = matchFirst(name, rarPats, zipPats, iszPats, ffsjPats) # xtremsplit pattern - r = pat17.search(name) + r = pat4.search(name) if r is not None: name = r.group(1) - # FFSJ pattern - for pattern in ffsjPats: - r = pattern.search(name) - if r is not None: - name = r.group(1) - break # remove part and cd pattern - r = pat12.search(name) + r = pat1.search(name) if r is not None: name = name.replace(r.group(0), "") - r = pat13.search(name) + r = pat2.search(name) if r is not None: name = name.replace(r.group(0), "") @@ -110,7 +98,7 @@ def parseNames(files): name = name[:-length] # remove endings like . _ - - r = pat14.search(name) + r = pat3.search(name) if r is not None: name = r.group(1) @@ -121,10 +109,10 @@ def parseNames(files): name = name.strip() # checks if name could be a hash - if file.find("file/"+name) >= 0: + if file.find("file/" + name) >= 0: name = "" - if file.find("files/"+name) >= 0: + if file.find("files/" + name) >= 0: name = "" r = re.search("^[0-9]+$", name, re.I) @@ -139,8 +127,9 @@ def parseNames(files): name = "" # fallback: package by hoster - if not len(name): + if not name: name = urlparse(file).hostname + if name: name = name.replace("ww.", "") # fallback : default name if not name: @@ -148,8 +137,26 @@ def parseNames(files): # build mapping if name in packs: - packs[name].append(file) + packs[name].append(url) else: - packs[name] = [file] - - return packs
\ No newline at end of file + packs[name] = [url] + + return packs + + +if __name__ == "__main__": + + from os.path import join + from pprint import pprint + + f = open(join("..", "..", "testlinks2.txt"), "rb") + urls = [(x.strip(), x.strip()) for x in f.readlines() if x.strip()] + f.close() + + print "Having %d urls." % len(urls) + + packs = parseNames(urls) + + pprint(packs) + + print "Got %d urls." % sum([len(x) for x in packs.itervalues()])
\ No newline at end of file diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index 4b55f0357..71c61942f 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -3,12 +3,13 @@ import re from module.plugins.Hoster import Hoster +from module.utils import html_unescape class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" __pattern__ = r"^unmatchable$" - __version__ = "0.1" + __version__ = "0.11" __description__ = """Base Plugin when any other didnt fit""" __author_name__ = ("RaNaN") __author_mail__ = ("RaNaN@pyload.org") @@ -38,7 +39,7 @@ class BasePlugin(Hoster): # return if pyfile.url.startswith("http"): - pyfile.name = re.findall("([^/=]+)", pyfile.url)[-1] + pyfile.name = html_unescape(re.findall("([^/=]+)", pyfile.url)[-1]) self.download(pyfile.url, disposition=True) else: diff --git a/module/remote/thriftbackend/pyload.thrift b/module/remote/thriftbackend/pyload.thrift index eff697d05..2995ce207 100644 --- a/module/remote/thriftbackend/pyload.thrift +++ b/module/remote/thriftbackend/pyload.thrift @@ -158,8 +158,8 @@ struct ServiceCall { } struct OnlineStatus { - 1: string url, - 2: string name, + 1: string name, + 2: PluginName plugin, 3: DownloadStatus status, 4: i64 size, // size <= 0 : unknown } @@ -206,10 +206,16 @@ service Pyload { bool toggleReconnect(), // download preparing + + // packagename - urls + map<string, LinkList> generatePackages(1: LinkList links), map<PluginName, LinkList> checkURLs(1: LinkList urls), map<PluginName, LinkList> parseURLs(1: string html), + + // parses results and generates packages ResultID checkOnlineStatus(1: LinkList urls), - map<PluginName, list<OnlineStatus>> pollResults(1: ResultID rid), + // poll results from previosly started online check , packagename - url - status + map<string, map<string, OnlineStatus>> pollResults(1: ResultID rid), // downloads - information list<DownloadInfo> statusDownloads(), @@ -224,6 +230,7 @@ service Pyload { map<i16, FileID> getFileOrder(1: PackageID pid) // downloads - adding/deleting + list<PackageID> generateAndAddPackages(1: LinkList links, 2: Destination dest), PackageID addPackage(1: string name, 2: LinkList links, 3: Destination dest), void addFiles(1: PackageID pid, 2: LinkList links), void uploadContainer(1: string filename, 2: binary data), diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote index 2b055321b..0c0e70bd4 100755 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote @@ -39,6 +39,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print ' bool isTimeDownload()' print ' bool isTimeReconnect()' print ' bool toggleReconnect()' + print ' generatePackages(LinkList links)' print ' checkURLs(LinkList urls)' print ' parseURLs(string html)' print ' ResultID checkOnlineStatus(LinkList urls)' @@ -53,6 +54,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print ' getCollectorData()' print ' getPackageOrder(Destination destination)' print ' getFileOrder(PackageID pid)' + print ' generateAndAddPackages(LinkList links, Destination dest)' print ' PackageID addPackage(string name, LinkList links, Destination dest)' print ' void addFiles(PackageID pid, LinkList links)' print ' void uploadContainer(string filename, string data)' @@ -235,6 +237,12 @@ elif cmd == 'toggleReconnect': sys.exit(1) pp.pprint(client.toggleReconnect()) +elif cmd == 'generatePackages': + if len(args) != 1: + print 'generatePackages requires 1 args' + sys.exit(1) + pp.pprint(client.generatePackages(eval(args[0]),)) + elif cmd == 'checkURLs': if len(args) != 1: print 'checkURLs requires 1 args' @@ -319,6 +327,12 @@ elif cmd == 'getFileOrder': sys.exit(1) pp.pprint(client.getFileOrder(eval(args[0]),)) +elif cmd == 'generateAndAddPackages': + if len(args) != 2: + print 'generateAndAddPackages requires 2 args' + sys.exit(1) + pp.pprint(client.generateAndAddPackages(eval(args[0]),eval(args[1]),)) + elif cmd == 'addPackage': if len(args) != 3: print 'addPackage requires 3 args' diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py index de3611b6d..009d850c2 100644 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py @@ -78,6 +78,13 @@ class Iface(object): def toggleReconnect(self, ): pass + def generatePackages(self, links): + """ + Parameters: + - links + """ + pass + def checkURLs(self, urls): """ Parameters: @@ -156,6 +163,14 @@ class Iface(object): """ pass + def generateAndAddPackages(self, links, dest): + """ + Parameters: + - links + - dest + """ + pass + def addPackage(self, name, links, dest): """ Parameters: @@ -821,6 +836,36 @@ class Client(Iface): return result.success raise TApplicationException(TApplicationException.MISSING_RESULT, "toggleReconnect failed: unknown result"); + def generatePackages(self, links): + """ + Parameters: + - links + """ + self.send_generatePackages(links) + return self.recv_generatePackages() + + def send_generatePackages(self, links): + self._oprot.writeMessageBegin('generatePackages', TMessageType.CALL, self._seqid) + args = generatePackages_args() + args.links = links + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_generatePackages(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = generatePackages_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "generatePackages failed: unknown result"); + def checkURLs(self, urls): """ Parameters: @@ -1222,6 +1267,38 @@ class Client(Iface): return result.success raise TApplicationException(TApplicationException.MISSING_RESULT, "getFileOrder failed: unknown result"); + def generateAndAddPackages(self, links, dest): + """ + Parameters: + - links + - dest + """ + self.send_generateAndAddPackages(links, dest) + return self.recv_generateAndAddPackages() + + def send_generateAndAddPackages(self, links, dest): + self._oprot.writeMessageBegin('generateAndAddPackages', TMessageType.CALL, self._seqid) + args = generateAndAddPackages_args() + args.links = links + args.dest = dest + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_generateAndAddPackages(self, ): + (fname, mtype, rseqid) = self._iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(self._iprot) + self._iprot.readMessageEnd() + raise x + result = generateAndAddPackages_result() + result.read(self._iprot) + self._iprot.readMessageEnd() + if result.success is not None: + return result.success + raise TApplicationException(TApplicationException.MISSING_RESULT, "generateAndAddPackages failed: unknown result"); + def addPackage(self, name, links, dest): """ Parameters: @@ -2286,6 +2363,7 @@ class Processor(Iface, TProcessor): self._processMap["isTimeDownload"] = Processor.process_isTimeDownload self._processMap["isTimeReconnect"] = Processor.process_isTimeReconnect self._processMap["toggleReconnect"] = Processor.process_toggleReconnect + self._processMap["generatePackages"] = Processor.process_generatePackages self._processMap["checkURLs"] = Processor.process_checkURLs self._processMap["parseURLs"] = Processor.process_parseURLs self._processMap["checkOnlineStatus"] = Processor.process_checkOnlineStatus @@ -2300,6 +2378,7 @@ class Processor(Iface, TProcessor): self._processMap["getCollectorData"] = Processor.process_getCollectorData self._processMap["getPackageOrder"] = Processor.process_getPackageOrder self._processMap["getFileOrder"] = Processor.process_getFileOrder + self._processMap["generateAndAddPackages"] = Processor.process_generateAndAddPackages self._processMap["addPackage"] = Processor.process_addPackage self._processMap["addFiles"] = Processor.process_addFiles self._processMap["uploadContainer"] = Processor.process_uploadContainer @@ -2528,6 +2607,17 @@ class Processor(Iface, TProcessor): oprot.writeMessageEnd() oprot.trans.flush() + def process_generatePackages(self, seqid, iprot, oprot): + args = generatePackages_args() + args.read(iprot) + iprot.readMessageEnd() + result = generatePackages_result() + result.success = self._handler.generatePackages(args.links) + oprot.writeMessageBegin("generatePackages", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_checkURLs(self, seqid, iprot, oprot): args = checkURLs_args() args.read(iprot) @@ -2691,6 +2781,17 @@ class Processor(Iface, TProcessor): oprot.writeMessageEnd() oprot.trans.flush() + def process_generateAndAddPackages(self, seqid, iprot, oprot): + args = generateAndAddPackages_args() + args.read(iprot) + iprot.readMessageEnd() + result = generateAndAddPackages_result() + result.success = self._handler.generateAndAddPackages(args.links, args.dest) + oprot.writeMessageBegin("generateAndAddPackages", TMessageType.REPLY, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_addPackage(self, seqid, iprot, oprot): args = addPackage_args() args.read(iprot) @@ -3535,6 +3636,43 @@ class toggleReconnect_result(TBase): self.success = success +class generatePackages_args(TBase): + """ + Attributes: + - links + """ + + __slots__ = [ + 'links', + ] + + thrift_spec = ( + None, # 0 + (1, TType.LIST, 'links', (TType.STRING,None), None, ), # 1 + ) + + def __init__(self, links=None,): + self.links = links + + +class generatePackages_result(TBase): + """ + Attributes: + - success + """ + + __slots__ = [ + 'success', + ] + + thrift_spec = ( + (0, TType.MAP, 'success', (TType.STRING,None,TType.LIST,(TType.STRING,None)), None, ), # 0 + ) + + def __init__(self, success=None,): + self.success = success + + class checkURLs_args(TBase): """ Attributes: @@ -3676,7 +3814,7 @@ class pollResults_result(TBase): ] thrift_spec = ( - (0, TType.MAP, 'success', (TType.STRING,None,TType.LIST,(TType.STRUCT,(OnlineStatus, OnlineStatus.thrift_spec))), None, ), # 0 + (0, TType.MAP, 'success', (TType.STRING,None,TType.MAP,(TType.STRING,None,TType.STRUCT,(OnlineStatus, OnlineStatus.thrift_spec))), None, ), # 0 ) def __init__(self, success=None,): @@ -4015,6 +4153,47 @@ class getFileOrder_result(TBase): self.success = success +class generateAndAddPackages_args(TBase): + """ + Attributes: + - links + - dest + """ + + __slots__ = [ + 'links', + 'dest', + ] + + thrift_spec = ( + None, # 0 + (1, TType.LIST, 'links', (TType.STRING,None), None, ), # 1 + (2, TType.I32, 'dest', None, None, ), # 2 + ) + + def __init__(self, links=None, dest=None,): + self.links = links + self.dest = dest + + +class generateAndAddPackages_result(TBase): + """ + Attributes: + - success + """ + + __slots__ = [ + 'success', + ] + + thrift_spec = ( + (0, TType.LIST, 'success', (TType.I32,None), None, ), # 0 + ) + + def __init__(self, success=None,): + self.success = success + + class addPackage_args(TBase): """ Attributes: diff --git a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py index fcff55ed8..f7052bc28 100644 --- a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py +++ b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py @@ -604,30 +604,30 @@ class ServiceCall(TBase): class OnlineStatus(TBase): """ Attributes: - - url - name + - plugin - status - size """ __slots__ = [ - 'url', 'name', + 'plugin', 'status', 'size', ] thrift_spec = ( None, # 0 - (1, TType.STRING, 'url', None, None, ), # 1 - (2, TType.STRING, 'name', None, None, ), # 2 + (1, TType.STRING, 'name', None, None, ), # 1 + (2, TType.STRING, 'plugin', None, None, ), # 2 (3, TType.I32, 'status', None, None, ), # 3 (4, TType.I64, 'size', None, None, ), # 4 ) - def __init__(self, url=None, name=None, status=None, size=None,): - self.url = url + def __init__(self, name=None, plugin=None, status=None, size=None,): self.name = name + self.plugin = plugin self.status = status self.size = size |