From d35c003cc53d4723d1dfe0d81eeb9bea78cee594 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 31 Dec 2011 16:01:24 +0100 Subject: new crypter plugin API, now decrypting possible for now. --- module/threads/InfoThread.py | 215 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 module/threads/InfoThread.py (limited to 'module/threads/InfoThread.py') diff --git a/module/threads/InfoThread.py b/module/threads/InfoThread.py new file mode 100644 index 000000000..4cba7da38 --- /dev/null +++ b/module/threads/InfoThread.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from time import time +from traceback import print_exc + +from module.Api import OnlineStatus +from module.PyFile import PyFile +from module.common.packagetools import parseNames + +from BaseThread import BaseThread + +class InfoThread(BaseThread): + def __init__(self, manager, data, pid=-1, rid=-1, add=False): + """Constructor""" + BaseThread.__init__(self, manager) + + self.data = data + self.pid = pid # package id + # [ .. (name, plugin) .. ] + + self.rid = rid #result id + self.add = add #add packages instead of return result + + self.cache = [] #accumulated data + + self.start() + + def run(self): + """run method""" + + plugins = {} + container = [] + + for url, plugin in self.data: + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + + # filter out container plugins + for name in self.m.core.pluginManager.getPlugins("container"): + if name in plugins: + container.extend([(name, url) for url in plugins[name]]) + + del plugins[name] + + #directly write to database + if self.pid > -1: + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) + self.m.core.files.save() + + elif self.add: + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateCache(pluginname, result) + + packs = parseNames([(name, url) for name, x, y, url in self.cache]) + + self.m.log.debug("Fetched and generated %d packages" % len(packs)) + + for k, v in packs: + self.m.core.api.addPackage(k, v) + + #empty cache + del self.cache[:] + + else: #post the results + + + for name, url in container: + #attach container content + try: + data = self.decryptContainer(name, url) + except: + print_exc() + self.m.log.error("Could not decrypt container.") + data = [] + + for url, plugin in data: + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + self.m.infoResults[self.rid] = {} + + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) + + #force to process cache + if self.cache: + self.updateResult(pluginname, [], True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateResult(pluginname, result, True) + + self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} + + self.m.timestamp = time() + 5 * 60 + + + def updateDB(self, plugin, result): + self.m.core.files.updateFileInfo(result, self.pid) + + def updateResult(self, plugin, result, force=False): + #parse package name and generate result + #accumulate results + + self.cache.extend(result) + + if len(self.cache) >= 20 or force: + #used for package generating + tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) + for name, size, status, url in self.cache] + + data = parseNames(tmp) + result = {} + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + self.m.setInfoResults(self.rid, result) + + self.cache = [] + + def updateCache(self, plugin, result): + self.cache.extend(result) + + def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): + try: + result = [] #result loaded from cache + process = [] #urls to process + for url in urls: + if url in self.m.infoCache: + result.append(self.m.infoCache[url]) + else: + process.append(url) + + if result: + self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) + cb(pluginname, result) + + if process: + self.m.log.debug("Run Info Fetching for %s" % pluginname) + for result in plugin.getInfo(process): + #result = [ .. (name, size, status, url) .. ] + if not type(result) == list: result = [result] + + for res in result: + self.m.infoCache[res[3]] = res + + cb(pluginname, result) + + self.m.log.debug("Finished Info Fetching for %s" % pluginname) + except Exception, e: + self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % + {"name": pluginname, "err": str(e)}) + if self.m.core.debug: + print_exc() + + # generate default results + if err: + result = [(url, 0, 3, url) for url in urls] + cb(pluginname, result) + + + def decryptContainer(self, plugin, url): + data = [] + # only works on container plugins + + self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) + + # dummy pyfile + pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) + + pyfile.initPlugin() + + # little plugin lifecycle + try: + pyfile.plugin.setup() + pyfile.plugin.loadToDisk() + pyfile.plugin.decrypt(pyfile) + pyfile.plugin.deleteTmp() + + for pack in pyfile.plugin.packages: + pyfile.plugin.urls.extend(pack[1]) + + data, crypter = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) + + self.m.log.debug("Got %d links." % len(data)) + + except Exception, e: + self.m.log.debug("Pre decrypting error: %s" % str(e)) + finally: + pyfile.release() + + return data -- cgit v1.2.3 From 35742c2cb023ac49ab3056752d2040cdb030cc2b Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 1 Jan 2012 13:36:59 +0100 Subject: Happy new Year ! --- module/threads/InfoThread.py | 51 +++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 32 deletions(-) (limited to 'module/threads/InfoThread.py') diff --git a/module/threads/InfoThread.py b/module/threads/InfoThread.py index 4cba7da38..596153c4b 100644 --- a/module/threads/InfoThread.py +++ b/module/threads/InfoThread.py @@ -7,11 +7,12 @@ from traceback import print_exc from module.Api import OnlineStatus from module.PyFile import PyFile from module.common.packagetools import parseNames +from module.utils import has_method from BaseThread import BaseThread class InfoThread(BaseThread): - def __init__(self, manager, data, pid=-1, rid=-1, add=False): + def __init__(self, manager, data, pid=-1, rid=-1): """Constructor""" BaseThread.__init__(self, manager) @@ -20,7 +21,6 @@ class InfoThread(BaseThread): # [ .. (name, plugin) .. ] self.rid = rid #result id - self.add = add #add packages instead of return result self.cache = [] #accumulated data @@ -39,8 +39,8 @@ class InfoThread(BaseThread): plugins[plugin] = [url] - # filter out container plugins - for name in self.m.core.pluginManager.getPlugins("container"): + # filter out crypter plugins + for name in self.m.core.pluginManager.getPlugins("crypter"): if name in plugins: container.extend([(name, url) for url in plugins[name]]) @@ -50,35 +50,17 @@ class InfoThread(BaseThread): if self.pid > -1: for pluginname, urls in plugins.iteritems(): plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): + klass = getattr(plugin, pluginname) + if has_method(klass, "getInfo"): + self.fetchForPlugin(pluginname, klass, urls, self.updateDB) + self.m.core.files.save() + elif has_method(plugin, "getInfo"): + self.log.debug("Deprecated .getInfo() method on module level, use classmethod instead") self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) self.m.core.files.save() - elif self.add: - for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) - - else: - #generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateCache(pluginname, result) - - packs = parseNames([(name, url) for name, x, y, url in self.cache]) - - self.m.log.debug("Fetched and generated %d packages" % len(packs)) - - for k, v in packs: - self.m.core.api.addPackage(k, v) - - #empty cache - del self.cache[:] - else: #post the results - - + #TODO: finer crypter control for name, url in container: #attach container content try: @@ -98,13 +80,18 @@ class InfoThread(BaseThread): for pluginname, urls in plugins.iteritems(): plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - if hasattr(plugin, "getInfo"): + klass = getattr(plugin, pluginname) + if has_method(klass, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) + #force to process cache + if self.cache: + self.updateResult(pluginname, [], True) + elif has_method(plugin, "getInfo"): + self.log.debug("Deprecated .getInfo() method on module level, use staticmethod instead") self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - #force to process cache if self.cache: self.updateResult(pluginname, [], True) - else: #generate default result result = [(url, 0, 3, url) for url in urls] -- cgit v1.2.3 From b877847094b0ba03a098dff0fd769eb456b48dd1 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Fri, 6 Jan 2012 17:54:53 +0100 Subject: several improvements, also closes #486, #487 --- module/threads/InfoThread.py | 48 ++++++++++++-------------------------------- 1 file changed, 13 insertions(+), 35 deletions(-) (limited to 'module/threads/InfoThread.py') diff --git a/module/threads/InfoThread.py b/module/threads/InfoThread.py index 596153c4b..c1e4458ef 100644 --- a/module/threads/InfoThread.py +++ b/module/threads/InfoThread.py @@ -30,7 +30,7 @@ class InfoThread(BaseThread): """run method""" plugins = {} - container = [] + crypter = {} for url, plugin in self.data: if plugin in plugins: @@ -42,8 +42,7 @@ class InfoThread(BaseThread): # filter out crypter plugins for name in self.m.core.pluginManager.getPlugins("crypter"): if name in plugins: - container.extend([(name, url) for url in plugins[name]]) - + crypter[name] = plugins[name] del plugins[name] #directly write to database @@ -60,11 +59,10 @@ class InfoThread(BaseThread): self.m.core.files.save() else: #post the results - #TODO: finer crypter control - for name, url in container: + for name, urls in crypter: #attach container content try: - data = self.decryptContainer(name, url) + data = self.decrypt(name, urls) except: print_exc() self.m.log.error("Could not decrypt container.") @@ -169,34 +167,14 @@ class InfoThread(BaseThread): cb(pluginname, result) - def decryptContainer(self, plugin, url): - data = [] - # only works on container plugins - - self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) - - # dummy pyfile - pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) - - pyfile.initPlugin() - - # little plugin lifecycle - try: - pyfile.plugin.setup() - pyfile.plugin.loadToDisk() - pyfile.plugin.decrypt(pyfile) - pyfile.plugin.deleteTmp() - - for pack in pyfile.plugin.packages: - pyfile.plugin.urls.extend(pack[1]) - - data, crypter = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) + def decrypt(self, plugin, urls): + self.m.log.debug("Pre decrypting %s" % plugin) + klass = self.m.core.pluginManager.loadClass("crypter", plugin) - self.m.log.debug("Got %d links." % len(data)) - - except Exception, e: - self.m.log.debug("Pre decrypting error: %s" % str(e)) - finally: - pyfile.release() + # only decrypt files + if has_method(klass, "decryptFile"): + urls = p.decrypt(urls) + data, crypter = self.m.core.pluginManager.parseUrls(urls) + return data - return data + return [] -- cgit v1.2.3 From 6eaa7bb25e2254c80c43fe46166142d590e86c64 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 7 Jan 2012 23:58:28 +0100 Subject: some cleanups --- module/threads/InfoThread.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) (limited to 'module/threads/InfoThread.py') diff --git a/module/threads/InfoThread.py b/module/threads/InfoThread.py index c1e4458ef..5f21d487c 100644 --- a/module/threads/InfoThread.py +++ b/module/threads/InfoThread.py @@ -5,9 +5,8 @@ from time import time from traceback import print_exc from module.Api import OnlineStatus -from module.PyFile import PyFile from module.common.packagetools import parseNames -from module.utils import has_method +from module.utils import has_method, accumulate from BaseThread import BaseThread @@ -29,16 +28,9 @@ class InfoThread(BaseThread): def run(self): """run method""" - plugins = {} + plugins = accumulate(self.data) crypter = {} - for url, plugin in self.data: - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] - - # filter out crypter plugins for name in self.m.core.pluginManager.getPlugins("crypter"): if name in plugins: @@ -68,11 +60,7 @@ class InfoThread(BaseThread): self.m.log.error("Could not decrypt container.") data = [] - for url, plugin in data: - if plugin in plugins: - plugins[plugin].append(url) - else: - plugins[plugin] = [url] + accumulate(data, plugins) self.m.infoResults[self.rid] = {} -- cgit v1.2.3 From bac28b7740aae772636d8b90e291d9c17dfd59a7 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 8 Jan 2012 14:44:59 +0100 Subject: new MultiHoster hook --- module/threads/InfoThread.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/threads/InfoThread.py') diff --git a/module/threads/InfoThread.py b/module/threads/InfoThread.py index 5f21d487c..7db85803a 100644 --- a/module/threads/InfoThread.py +++ b/module/threads/InfoThread.py @@ -40,8 +40,8 @@ class InfoThread(BaseThread): #directly write to database if self.pid > -1: for pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(pluginname, True) - klass = getattr(plugin, pluginname) + plugin = self.m.core.pluginManager.getPluginModule(pluginname) + klass = self.m.core.pluginManager.getPluginClass(pluginname) if has_method(klass, "getInfo"): self.fetchForPlugin(pluginname, klass, urls, self.updateDB) self.m.core.files.save() -- cgit v1.2.3 From 4df2b77fdf42046fe19bd371be7c7255986b5980 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 6 Mar 2012 13:36:39 +0100 Subject: renamed hooks to addons, new filemanager and database, many new api methods you will loose ALL your LINKS, webinterface will NOT work --- module/threads/InfoThread.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'module/threads/InfoThread.py') diff --git a/module/threads/InfoThread.py b/module/threads/InfoThread.py index 7db85803a..a8a2c6e7e 100644 --- a/module/threads/InfoThread.py +++ b/module/threads/InfoThread.py @@ -4,7 +4,7 @@ from time import time from traceback import print_exc -from module.Api import OnlineStatus +from module.Api import LinkStatus from module.common.packagetools import parseNames from module.utils import has_method, accumulate @@ -100,7 +100,7 @@ class InfoThread(BaseThread): if len(self.cache) >= 20 or force: #used for package generating - tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) + tmp = [(name, (url, LinkStatus(name, plugin, "unknown", status, int(size)))) for name, size, status, url in self.cache] data = parseNames(tmp) @@ -161,7 +161,7 @@ class InfoThread(BaseThread): # only decrypt files if has_method(klass, "decryptFile"): - urls = p.decrypt(urls) + urls = klass.decrypt(urls) data, crypter = self.m.core.pluginManager.parseUrls(urls) return data -- cgit v1.2.3