diff options
| -rw-r--r-- | module/FileDatabase.py | 135 | ||||
| -rw-r--r-- | module/HookManager.py | 4 | ||||
| -rw-r--r-- | module/PluginThread.py | 47 | ||||
| -rw-r--r-- | module/ThreadManager.py | 32 | ||||
| -rw-r--r-- | module/plugins/hoster/RapidshareCom.py | 36 |
5 files changed, 114 insertions, 140 deletions
diff --git a/module/FileDatabase.py b/module/FileDatabase.py index 507b5ae7f..98d75d58e 100644 --- a/module/FileDatabase.py +++ b/module/FileDatabase.py @@ -81,7 +81,6 @@ class FileHandler: #@TODO: purge the cache self.jobCache = {} - self.noNewInfoJobs = False self.lock = RLock() @@ -129,15 +128,20 @@ class FileHandler: @change def addLinks(self, urls, package): """adds links""" - + + data = [] + for x in self.core.pluginManager.parseUrls(urls): # tuple of (url, name, plugin, package) lastID = self.db.addLink(x[0], x[0], x[1], package) - self.noNewInfoJobs = False + data.append( (x[0],x[1] )) + f = self.db.getFile(lastID) e = InsertEvent("file", lastID, f.order, "collector" if not self.getPackage(package).queue else "queue") self.core.pullManager.addEvent(e) + f.release() + self.core.threadManager.createInfoThread(data) #---------------------------------------------------------------------- @change @@ -307,16 +311,7 @@ class FileHandler: self.lock.release() return pyfile - - def getInfoJob(self): - if self.noNewInfoJobs: - return None - jobs = self.db.getInfoJob() - if not jobs: - self.noNewInfoJobs = True - return None - return self.getFile(jobs[0]) - + #---------------------------------------------------------------------- def getFileCount(self): """returns number of files""" @@ -407,6 +402,11 @@ class FileHandler: e = ReloadAllEvent("collector" if not f.package().queue else "queue") self.core.pullManager.addEvent(e) + + @change + def updateFileInfo(self, data): + """ updates file info (name, size, status, url)""" + self.db.updateLinkInfo(data) ######################################################################### class FileDatabaseBackend(Thread): @@ -502,17 +502,15 @@ class FileDatabaseBackend(Thread): def _nextPackageOrder(self, queue=0): self.c.execute('SELECT packageorder FROM packages WHERE queue=?', (queue, )) - rs = self.c.fetchall() o = -1 - for r in rs: + for r in self.c: if r[0] > o: o = r[0] return o+1 def _nextFileOrder(self, package): self.c.execute('SELECT linkorder FROM links WHERE package=?', (package, )) - rs = self.c.fetchall() o = -1 - for r in rs: + for r in self.c: if r[0] > o: o = r[0] return o+1 @@ -648,7 +646,12 @@ class FileDatabaseBackend(Thread): @queue def updatePackage(self, p): self.c.execute('UPDATE packages SET name=?,folder=?,site=?,password=?,queue=?,priority=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, p.priority, str(p.id))) - + + @async + def updateLinkInfo(self, data): + """ data is list of tupels (name, size, status, url) """ + self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=?', data) + @queue def reorderPackage(self, p, position, noMove=False): if position == -1: @@ -723,17 +726,6 @@ class FileDatabaseBackend(Thread): return [x[0] for x in self.c ] - - @queue - def getInfoJob(self): - """return pyfile instance, which is suitable for info grabbing""" - - cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.url = l.name ORDER BY l.linkorder LIMIT 5" - - self.c.execute(cmd) # very bad! - - return [x[0] for x in self.c ] - class PyFile(): def __init__(self, manager, id, url, name, size, status, error, pluginname, package, order): self.m = manager @@ -973,7 +965,8 @@ class PyPackage(): if __name__ == "__main__": pypath = "." - + _ = lambda x : x + db = FileHandler(None) #p = PyFile(db, 5) @@ -983,75 +976,39 @@ if __name__ == "__main__": #print db.addPackage("package", "folder" , 1) - #print db.addPackage("package", "folder", 1) + pack = db.db.addPackage("package", "folder", 1) + + updates = [] + + + for x in range(0, 200): + x = str(x) + db.db.addLink("http://somehost.com/hoster/file/download?file_id="+x,x,"BasePlugin", pack) + updates.append( ("new name"+x,0,3, "http://somehost.com/hoster/file/download?file_id="+x) ) - #db.addLinks([x for x in range(0,200)], 5) - db.save() + for x in range(0, 100): + updates.append( ("unimportant%s"%x, 0, 3 , "a really long non existent url%s" %x ) ) + + db.db.commit() b = time() print "adding 200 links, single sql execs, no commit", b-a - - - res = db.getCompleteData(1) - #print res - r = [ len(x["links"]) for x in res.itervalues() ] - print r - c = time() - print "get all links", c-b - - #i = 2 - #db.updateLink(i, "urlupdate%s" % i, "nameupdate%s" % i, i, i, i,i) - - d = time() - print "update one link", d-c - - #p.sync() - #p.remove() - - e = time() - print "sync and remove link", e-d - - db.save() - - db.deletePackage(1) - #db.commit() - - f = time() - print "commit, remove package/links, commit", f-e - - #db.commit() - sleep(0.5) - - g = time() - print "empty commit", g-f -0.5 - - - job = db.getJob("") - print job - h = time() - #print db.getFileCount() + print db.getCompleteData(1) + + c = time() - print "get job", h-g - print db.getFileCount() - - i = time() - - print "file count", i-h - + db.db.updateLinkInfo(updates) - print db.getJob("") - - j = time() - - - print "get job 2", j-i + d = time() - for i in db.cache.itervalues(): - i.sync() + print "updates", d-c + + print db.getCompleteData(1) - sleep(1) + e = time() + print "complete data", e-d
\ No newline at end of file diff --git a/module/HookManager.py b/module/HookManager.py index 503b9f77d..7cd903403 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -49,8 +49,8 @@ class HookManager(): try: #hookClass = getattr(plugin, plugin.__name__) - plugin = pluginClass(self.core) - if plugin.getConfig("load"): + if self.core.config.getPlugin(pluginClass.__name__, "load"): + plugin = pluginClass(self.core) plugins.append(plugin) self.log.info(_("%s loaded, activated %s") % (pluginClass.__name__, plugin.isActivated() )) except: diff --git a/module/PluginThread.py b/module/PluginThread.py index d17f638f0..592fc7521 100644 --- a/module/PluginThread.py +++ b/module/PluginThread.py @@ -267,34 +267,37 @@ class HookThread(PluginThread): class InfoThread(PluginThread): #---------------------------------------------------------------------- - def __init__(self, manager): + def __init__(self, manager, data): """Constructor""" PluginThread.__init__(self, manager) - self.queue = Queue() # job queue - self.active = False - + self.data = data + # [ .. (name, plugin) .. ] self.start() #---------------------------------------------------------------------- def run(self): """run method""" - while True: - self.active = self.queue.get() - if self.active == "quit": - return True - pyfile = self.active - - pyfile.plugin.getInfo() - - #---------------------------------------------------------------------- - def put(self, job): - """assing job to thread""" - self.queue.put(job) - - #---------------------------------------------------------------------- - def stop(self): - """stops the thread""" - self.put("quit") - + plugins = {} + + for url, plugin in self.data: + if plugins.has_key(plugin): + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(plugin) + if hasattr(plugin, "getInfo"): + print "get", urls + print "" + for result in plugin.getInfo(urls): + if not type(result) == list: result = [result] + print "updating", result + print "" + self.m.core.files.updateFileInfo(result) + + print "" + print "finished info fetching" + self.m.core.files.save()
\ No newline at end of file diff --git a/module/ThreadManager.py b/module/ThreadManager.py index 1e4b8ac2b..9e8da1b2c 100644 --- a/module/ThreadManager.py +++ b/module/ThreadManager.py @@ -41,7 +41,7 @@ class ThreadManager: self.threads = [] # thread list self.localThreads = [] #hook+decrypter threads - self.infoThread = PluginThread.InfoThread(self) + #self.infoThread = PluginThread.InfoThread(self) self.pause = True @@ -61,6 +61,16 @@ class ThreadManager: self.threads.append(thread) #---------------------------------------------------------------------- + def createInfoThread(self, data): + """ + start a thread whichs fetches online status and other infos + data = [ .. () .. ] + """ + + PluginThread.InfoThread(self, data) + + + #---------------------------------------------------------------------- def downloadingIds(self): """get a list of the currently downloading pyfile's ids""" return [x.active.id for x in self.threads if x.active and x.active != "quit"] @@ -174,22 +184,4 @@ class ThreadManager: else: thread = PluginThread.DecrypterThread(self, job) - - job = self.core.files.getInfoJob() - if job: - try: - job.initPlugin() - except Exception, e: - self.log.critical(str(e)) - if self.core.debug: - print_exc() - - if job.plugin.__type__ == "hoster": - self.infoThread.put(job) - - - - - - - +
\ No newline at end of file diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py index 07dc3ed21..bf94e3702 100644 --- a/module/plugins/hoster/RapidshareCom.py +++ b/module/plugins/hoster/RapidshareCom.py @@ -5,9 +5,37 @@ import re from time import time +from module.network.Request import getURL from module.plugins.Hoster import Hoster import hashlib +def getInfo(urls): + + ids = "" + names = "" + + for url in urls: + tmp = url.split("/") + ids+= ","+tmp[-2] + names+= ","+tmp[-1] + + url = "http://api.rapidshare.com/cgi-bin/rsapi.cgi?sub=checkfiles_v1&files=%s&filenames=%s" % (ids[1:], names[1:]) + + + api = getURL(url) + result = [] + i = 0 + for res in api.split(): + tmp = res.split(",") + if tmp[4] in ("0", "4", "5"): status = 1 + elif tmp[4] == "1": status = 2 + else: status = 3 + + result.append( (tmp[1], tmp[2], status, urls[i]) ) + i += 1 + + yield result + class RapidshareCom(Hoster): __name__ = "RapidshareCom" __type__ = "hoster" @@ -31,13 +59,7 @@ class RapidshareCom(Hoster): self.url = self.pyfile.url self.prepare() self.proceed(self.url) - - def getInfo(self): - self.url = self.pyfile.url - self.download_api_data() - self.pyfile.name = self.api_data["filename"] - self.pyfile.sync() - + def prepare(self): # self.no_slots = True # self.want_reconnect = False |