better link checking

5 files changed, 114 insertions, 140 deletions

diff --git a/module/FileDatabase.py b/module/FileDatabase.py
index 507b5ae7f..98d75d58e 100644
--- a/module/FileDatabase.py
+++ b/module/FileDatabase.py
@@ -81,7 +81,6 @@ class FileHandler:
         #@TODO: purge the cache
 
         self.jobCache = {}
-        self.noNewInfoJobs = False
         
         self.lock = RLock()
         
@@ -129,15 +128,20 @@ class FileHandler:
     @change
     def addLinks(self, urls, package):
         """adds links"""
-    
+       
+        data = []
+        
         for x in self.core.pluginManager.parseUrls(urls):
             # tuple of (url, name, plugin, package)
             lastID = self.db.addLink(x[0], x[0], x[1], package)
-            self.noNewInfoJobs = False
+            data.append( (x[0],x[1] ))
+            
             f = self.db.getFile(lastID)
             e = InsertEvent("file", lastID, f.order, "collector" if not self.getPackage(package).queue else "queue")
             self.core.pullManager.addEvent(e)
+            f.release()
         
+        self.core.threadManager.createInfoThread(data)
 
     #----------------------------------------------------------------------
     @change
@@ -307,16 +311,7 @@ class FileHandler:
         
         self.lock.release()
         return pyfile
-    
-    def getInfoJob(self):
-        if self.noNewInfoJobs:
-            return None
-        jobs = self.db.getInfoJob()
-        if not jobs:
-            self.noNewInfoJobs = True
-            return None
-        return self.getFile(jobs[0])
-    
+
     #----------------------------------------------------------------------
     def getFileCount(self):
         """returns number of files"""
@@ -407,6 +402,11 @@ class FileHandler:
         
         e = ReloadAllEvent("collector" if not f.package().queue else "queue")
         self.core.pullManager.addEvent(e)
+        
+    @change
+    def updateFileInfo(self, data):
+        """ updates file info (name, size, status, url)"""
+        self.db.updateLinkInfo(data)
 
 #########################################################################
 class FileDatabaseBackend(Thread):
@@ -502,17 +502,15 @@ class FileDatabaseBackend(Thread):
     
     def _nextPackageOrder(self, queue=0):
         self.c.execute('SELECT packageorder FROM packages WHERE queue=?', (queue, ))
-        rs = self.c.fetchall()
         o = -1
-        for r in rs:
+        for r in self.c:
             if r[0] > o: o = r[0]
         return o+1
     
     def _nextFileOrder(self, package):
         self.c.execute('SELECT linkorder FROM links WHERE package=?', (package, ))
-        rs = self.c.fetchall()
         o = -1
-        for r in rs:
+        for r in self.c:
             if r[0] > o: o = r[0]
         return o+1
     
@@ -648,7 +646,12 @@ class FileDatabaseBackend(Thread):
     @queue
     def updatePackage(self, p):
         self.c.execute('UPDATE packages SET name=?,folder=?,site=?,password=?,queue=?,priority=? WHERE id=?', (p.name, p.folder, p.site, p.password, p.queue, p.priority, str(p.id)))
-    
+        
+    @async    
+    def updateLinkInfo(self, data):
+        """ data is list of tupels (name, size, status, url) """
+        self.c.executemany('UPDATE links SET name=?, size=?, status=? WHERE url=?', data)
+        
     @queue
     def reorderPackage(self, p, position, noMove=False):
         if position == -1:
@@ -723,17 +726,6 @@ class FileDatabaseBackend(Thread):
 
         return [x[0] for x in self.c ]
 
-
-    @queue
-    def getInfoJob(self):
-        """return pyfile instance, which is suitable for info grabbing"""
-        
-        cmd = "SELECT l.id FROM links as l INNER JOIN packages as p ON l.package=p.id WHERE l.url = l.name ORDER BY l.linkorder LIMIT 5"
-            
-        self.c.execute(cmd) # very bad!
-
-        return [x[0] for x in self.c ]
-
 class PyFile():
     def __init__(self, manager, id, url, name, size, status, error, pluginname, package, order):
         self.m = manager
@@ -973,7 +965,8 @@ class PyPackage():
 if __name__ == "__main__":
 
     pypath = "."
-
+    _ = lambda x : x
+    
     db = FileHandler(None)
 
     #p = PyFile(db, 5)
@@ -983,75 +976,39 @@ if __name__ == "__main__":
 
     #print db.addPackage("package", "folder" , 1)
     
-    #print db.addPackage("package", "folder",  1)
+    pack = db.db.addPackage("package", "folder",  1)
+    
+    updates = []
+    
+    
+    for x in range(0, 200):       
+        x = str(x)
+        db.db.addLink("http://somehost.com/hoster/file/download?file_id="+x,x,"BasePlugin", pack)
+        updates.append( ("new name"+x,0,3, "http://somehost.com/hoster/file/download?file_id="+x)  )
 
-    #db.addLinks([x for x in range(0,200)], 5)
 
-    db.save()
+    for x in range(0, 100):
+        updates.append( ("unimportant%s"%x, 0, 3 , "a really long non existent url%s" %x ) )
+        
+    db.db.commit()
 
     b = time()
     print "adding 200 links, single sql execs, no commit", b-a
-
-
-    res = db.getCompleteData(1)
-    #print res
-    r = [ len(x["links"]) for x in res.itervalues() ]
-    print r
-    c = time()
-    print "get all links", c-b
-
-    #i = 2
-    #db.updateLink(i, "urlupdate%s" % i, "nameupdate%s" % i, i, i, i,i)
-
-    d = time()
-    print "update one link", d-c
-
-    #p.sync()
-    #p.remove()
-
-    e = time()
-    print "sync and remove link", e-d
-
-    db.save()
-
-    db.deletePackage(1)
-    #db.commit()
-
-    f = time()
-    print "commit, remove package/links, commit", f-e
-
-    #db.commit()
-    sleep(0.5)
-
-    g = time()
-    print "empty commit", g-f -0.5
-
-
-    job = db.getJob("")
-    print job
     
-    h = time()
-    #print db.getFileCount()
+    print db.getCompleteData(1)
+ 
+    c  = time()
     
-    print "get job", h-g
 
-    print db.getFileCount()
-    
-    i = time()
-    
-    print "file count", i-h
-    
+    db.db.updateLinkInfo(updates)
     
-    print db.getJob("")
-    
-    j = time()
-    
-    
-    print "get job 2", j-i
+    d = time()
     
-    for i in db.cache.itervalues():
-        i.sync()
+    print "updates", d-c
+
+    print db.getCompleteData(1)
     
-    sleep(1)
     
+    e = time()
     
+    print "complete data", e-d
\ No newline at end of file
diff --git a/module/HookManager.py b/module/HookManager.py
index 503b9f77d..7cd903403 100644
--- a/module/HookManager.py
+++ b/module/HookManager.py
@@ -49,8 +49,8 @@ class HookManager():
             try:
                 #hookClass = getattr(plugin, plugin.__name__)
                 
-                plugin = pluginClass(self.core)
-                if plugin.getConfig("load"):
+                if self.core.config.getPlugin(pluginClass.__name__, "load"):
+                    plugin = pluginClass(self.core)
                     plugins.append(plugin)
                     self.log.info(_("%s loaded, activated %s") % (pluginClass.__name__, plugin.isActivated() ))
             except:
diff --git a/module/PluginThread.py b/module/PluginThread.py
index d17f638f0..592fc7521 100644
--- a/module/PluginThread.py
+++ b/module/PluginThread.py
@@ -267,34 +267,37 @@ class HookThread(PluginThread):
 class InfoThread(PluginThread):
 
 	#----------------------------------------------------------------------
-	def __init__(self, manager):
+	def __init__(self, manager, data):
 		"""Constructor"""
 		PluginThread.__init__(self, manager)
 		
-		self.queue = Queue() # job queue
-		self.active = False
-		
+		self.data = data
+		# [ .. (name, plugin) .. ]
 		self.start()
 		
 	#----------------------------------------------------------------------
 	def run(self):
 		"""run method"""
 		
-		while True:
-			self.active = self.queue.get()
-			if self.active == "quit":
-			    return True
-			pyfile = self.active
-			
-			pyfile.plugin.getInfo()
-			
-	#----------------------------------------------------------------------
-	def put(self, job):
-		"""assing job to thread"""
-		self.queue.put(job)
-	
-	#----------------------------------------------------------------------
-	def stop(self):
-		"""stops the thread"""
-		self.put("quit")
-	
+		plugins = {}
+		
+		for url, plugin in self.data:
+			if plugins.has_key(plugin):
+				plugins[plugin].append(url)
+			else:
+				plugins[plugin] = [url]
+
+		for pluginname, urls in plugins.iteritems():
+			plugin = self.m.core.pluginManager.getPlugin(plugin)
+			if hasattr(plugin, "getInfo"):
+				print "get", urls
+				print ""
+				for result in plugin.getInfo(urls):
+					if not type(result) == list: result = [result]
+					print "updating", result
+					print ""
+					self.m.core.files.updateFileInfo(result)
+		
+		print ""			
+		print "finished info fetching"
+		self.m.core.files.save()
\ No newline at end of file
diff --git a/module/ThreadManager.py b/module/ThreadManager.py
index 1e4b8ac2b..9e8da1b2c 100644
--- a/module/ThreadManager.py
+++ b/module/ThreadManager.py
@@ -41,7 +41,7 @@ class ThreadManager:
         self.threads = []  # thread list
         self.localThreads = []  #hook+decrypter threads
         
-        self.infoThread = PluginThread.InfoThread(self)
+        #self.infoThread = PluginThread.InfoThread(self)
                 
         self.pause = True
         
@@ -61,6 +61,16 @@ class ThreadManager:
         self.threads.append(thread)
         
     #----------------------------------------------------------------------
+    def createInfoThread(self, data):
+        """
+        start a thread whichs fetches online status and other infos
+        data = [ .. () .. ]
+        """
+        
+        PluginThread.InfoThread(self, data)
+        
+        
+    #----------------------------------------------------------------------
     def downloadingIds(self):
         """get a list of the currently downloading pyfile's ids"""
         return [x.active.id for x in self.threads if x.active and x.active != "quit"]
@@ -174,22 +184,4 @@ class ThreadManager:
                     
             else:
                 thread = PluginThread.DecrypterThread(self, job)
-        
-        job = self.core.files.getInfoJob()
-        if job:
-            try:
-                job.initPlugin()
-            except Exception, e:
-                self.log.critical(str(e))
-                if self.core.debug:
-                    print_exc()
-            
-            if job.plugin.__type__ == "hoster":
-                self.infoThread.put(job)
-                    
-    
-        
-        
-        
-    
-    
+        
\ No newline at end of file
diff --git a/module/plugins/hoster/RapidshareCom.py b/module/plugins/hoster/RapidshareCom.py
index 07dc3ed21..bf94e3702 100644
--- a/module/plugins/hoster/RapidshareCom.py
+++ b/module/plugins/hoster/RapidshareCom.py
@@ -5,9 +5,37 @@
 import re
 from time import time
 
+from module.network.Request import getURL
 from module.plugins.Hoster import Hoster
 import hashlib
 
+def getInfo(urls):
+    
+    ids = ""
+    names = ""
+    
+    for url in urls:
+        tmp = url.split("/")
+        ids+= ","+tmp[-2]
+        names+= ","+tmp[-1]
+    
+    url = "http://api.rapidshare.com/cgi-bin/rsapi.cgi?sub=checkfiles_v1&files=%s&filenames=%s" % (ids[1:], names[1:])
+    
+    
+    api = getURL(url)
+    result = []
+    i = 0
+    for res in api.split():
+        tmp = res.split(",")
+        if tmp[4] in ("0", "4", "5"): status = 1
+        elif tmp[4] == "1": status = 2
+        else: status = 3
+        
+        result.append( (tmp[1], tmp[2], status, urls[i]) ) 
+        i += 1
+        
+    yield result
+
 class RapidshareCom(Hoster):
     __name__ = "RapidshareCom"
     __type__ = "hoster"
@@ -31,13 +59,7 @@ class RapidshareCom(Hoster):
         self.url = self.pyfile.url        
         self.prepare()
         self.proceed(self.url)
-    
-    def getInfo(self):
-        self.url = self.pyfile.url
-        self.download_api_data()
-        self.pyfile.name = self.api_data["filename"]
-        self.pyfile.sync()
-     
+         
     def prepare(self):
         # self.no_slots = True
         # self.want_reconnect = False