mirror detection/handling

6 files changed, 96 insertions, 59 deletions

diff --git a/module/HookManager.py b/module/HookManager.py
index 9688d8c60..29756f8c5 100644
--- a/module/HookManager.py
+++ b/module/HookManager.py
@@ -109,8 +109,8 @@ class HookManager:
                 if self.core.debug:
                     traceback.print_exc()
 
-        self.log.info(_("Activated plugins: %s") % ", ".join(active))
-        self.log.info(_("Deactivate plugins: %s") % ", ".join(deactive))
+        self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active)))
+        self.log.info(_("Deactivate plugins: %s") % ", ".join(sorted(deactive)))
         #self.log.info(_("Not loaded plugins: %s") % ", ".join(unloaded))
 
         self.plugins = plugins
diff --git a/module/PluginThread.py b/module/PluginThread.py
index 7f967f989..33701ee69 100644
--- a/module/PluginThread.py
+++ b/module/PluginThread.py
@@ -143,7 +143,7 @@ class DownloadThread(PluginThread):
 
             try:
 
-                pyfile.plugin.checkForSameFiles()
+                pyfile.plugin.checkForSameFiles(starting=True)
                 self.m.log.info(_("Download starts: %s" % pyfile.name))
 
                 # start download
@@ -246,12 +246,18 @@ class DownloadThread(PluginThread):
 
                 pyfile.setStatus("skipped")
 
-                self.m.log.info(_("Download skipped: %(name)s") % pyfile.name)
-                if self.m.core.debug:
-                    self.m.log.debug("Skipped due to %s" % e.message)
+                self.m.log.info(_("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message})
 
                 self.clean(pyfile)
 
+                self.m.core.files.checkPackageFinished(pyfile)
+
+                self.active = False
+                pyfile.finishIfDone()
+                self.m.core.files.save()
+
+                continue
+
 
             except Exception, e:
                 pyfile.setStatus("failed")
diff --git a/module/database/FileDatabase.py b/module/database/FileDatabase.py
index 99812d2c9..3ba43f881 100644
--- a/module/database/FileDatabase.py
+++ b/module/database/FileDatabase.py
@@ -814,6 +814,13 @@ class FileMethods():
     def restartFailed(self):
         self.c.execute("UPDATE links SET status=3,error='' WHERE status IN (8, 9)")
 
+
+    @style.queue
+    def findDuplicates(self, id, pid, filename):
+        """ checks if filename exists with different id and same package """
+        self.c.execute("SELECT plugin FROM links where id!=? AND status=0 AND package=? AND name=?", (id, pid, filename))
+        return self.c.fetchone()
+
 DatabaseBackend.registerSub(FileMethods)
 
 if __name__ == "__main__":
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index c2e1c3044..cffbb8acb 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -40,34 +40,39 @@ from itertools import islice
 from module.utils import save_join, decode, removeChars
 
 def chunks(iterable, size):
-  it = iter(iterable)
-  item = list(islice(it, size))
-  while item:
-    yield item
+    it = iter(iterable)
     item = list(islice(it, size))
+    while item:
+        yield item
+        item = list(islice(it, size))
 
 
 class Abort(Exception):
     """ raised when aborted """
 
+
 class Fail(Exception):
     """ raised when failed """
-    
+
+
 class Reconnect(Exception):
     """ raised when reconnected """
 
+
 class Retry(Exception):
     """ raised when start again from beginning """
 
+
 class SkipDownload(Exception):
     """ raised when download should be skipped """
 
+
 class Plugin(object):
     __name__ = "Plugin"
     __version__ = "0.4"
     __pattern__ = None
     __type__ = "hoster"
-    __config__ = [ ("name", "type", "desc" , "default") ]
+    __config__ = [("name", "type", "desc", "default")]
     __description__ = """Base Plugin"""
     __author_name__ = ("RaNaN", "spoob", "mkaay")
     __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de")
@@ -101,7 +106,7 @@ class Plugin(object):
             self.premium = self.account.isPremium(self.user)  #premium status
         else:
             self.req = pyfile.m.core.requestFactory.getRequest(self.__name__)
-        
+
         self.log = pyfile.m.core.log
 
         self.pyfile = pyfile
@@ -115,12 +120,12 @@ class Plugin(object):
         self.html = None #some plugins store html code here
 
         self.init()
-    
+
     def getChunkCount(self):
         if self.chunkLimit <= 0:
             return self.config["download"]["chunks"]
         return min(self.config["download"]["chunks"], self.chunkLimit)
-    
+
     def __call__(self):
         return self.__name__
 
@@ -151,13 +156,13 @@ class Plugin(object):
     def process(self, pyfile):
         """the 'main' method of every plugin"""
         raise NotImplementedError
-    
+
     def resetAccount(self):
         """ dont use account and retry download """
         self.account = None
         self.req = self.core.requestFactory.getRequest(self.__name__)
         self.retry()
-    
+
     def checksum(self, local_file=None):
         """
         return codes:
@@ -181,14 +186,14 @@ class Plugin(object):
         return self.config.getPlugin(self.__name__, option)
 
     def setConfig(self, option, value):
-      """ sets a config value """
-      self.setConf(option, value)
-      
+        """ sets a config value """
+        self.setConf(option, value)
+
     def getConfig(self, option):
-      """ gets a config value """
-      return self.getConf(option)
-      
-      
+        """ gets a config value """
+        return self.getConf(option)
+
+
     def setWait(self, seconds, reconnect=False):
         """ set the wait time to specified seconds """
         if reconnect:
@@ -199,16 +204,16 @@ class Plugin(object):
         """ waits the time previously set """
         self.waiting = True
         self.pyfile.setStatus("waiting")
-        
+
         while self.pyfile.waitUntil > time():
             self.thread.m.reconnecting.wait(2)
-            
+
             if self.pyfile.abort: raise Abort
             if self.thread.m.reconnecting.isSet():
                 self.waiting = False
                 self.wantReconnect = False
                 raise Reconnect
-        
+
         self.waiting = False
         self.pyfile.setStatus("starting")
 
@@ -227,28 +232,28 @@ class Plugin(object):
     def retry(self):
         """ begin again from the beginning """
         raise Retry
-    
+
     def invalidCaptcha(self):
         if self.cTask:
             self.cTask.invalid()
-    
+
     def correctCaptcha(self):
         if self.cTask:
             self.cTask.correct()
 
     def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype="jpg"):
         """ loads the catpcha and decrypt it or ask the user for input """
-        
+
         content = self.load(url, get=get, post=post, cookies=cookies)
 
-        id = ("%.2f" % time())[-6:].replace(".","")
-        temp = open(join("tmp","tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb")
-        
+        id = ("%.2f" % time())[-6:].replace(".", "")
+        temp = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb")
+
         temp.write(content)
         temp.close()
 
         has_plugin = self.core.pluginManager.captchaPlugins.has_key(self.__name__)
-        
+
         if self.core.captcha:
             Ocr = self.core.pluginManager.getCaptchaPlugin(self.__name__)
         else:
@@ -257,16 +262,15 @@ class Plugin(object):
         if Ocr and not forceUser:
             sleep(randint(3000, 5000) / 1000.0)
             if self.pyfile.abort: raise Abort
-            
+
             ocr = Ocr()
             result = ocr.get_captcha(temp.name)
         else:
-            
             captchaManager = self.core.captchaManager
             task = captchaManager.newTask(content, imgtype, temp.name)
             self.cTask = task
             captchaManager.handleCaptcha(task)
-            
+
             while task.isWaiting():
                 if self.pyfile.abort:
                     captchaManager.removeTask(task)
@@ -282,16 +286,15 @@ class Plugin(object):
             elif not task.result:
                 self.fail(_("No captcha result obtained in appropiate time by any of the plugins."))
 
-
             result = task.result
             self.log.debug("Received captcha result: %s" % result)
 
         if not self.core.debug:
-          try:
-            remove(temp.name)
-          except:
-            pass
-        
+            try:
+                remove(temp.name)
+            except:
+                pass
+
         return result
 
 
@@ -307,25 +310,27 @@ class Plugin(object):
 
         if self.core.debug:
             from inspect import currentframe
+
             frame = currentframe()
             if not exists(join("tmp", self.__name__)):
                 makedirs(join("tmp", self.__name__))
 
-            f = open(join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)), "wb")
+            f = open(
+                join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
+                , "wb")
             del frame # delete the frame or it wont be cleaned
-            
+
             try:
                 tmp = res.encode("utf8")
             except:
                 tmp = res
-            
+
             f.write(tmp)
             f.close()
 
-
         if just_header:
             #parse header
-            header = {"code" : self.req.code}
+            header = {"code": self.req.code}
             for line in res.splitlines():
                 line = line.strip()
                 if not line or ":" not in line: continue
@@ -338,7 +343,7 @@ class Plugin(object):
                     if type(header[key]) == list:
                         header[key].append(value)
                     else:
-                        header[key] = [header[key],value]
+                        header[key] = [header[key], value]
                 else:
                     header[key] = value
             res = header
@@ -353,11 +358,11 @@ class Plugin(object):
         self.pyfile.setStatus("downloading")
 
         download_folder = self.config['general']['download_folder']
-        
+
         location = save_join(download_folder, self.pyfile.package().folder)
 
         if not exists(location):
-            makedirs(location, int(self.core.config["permission"]["folder"],8))
+            makedirs(location, int(self.core.config["permission"]["folder"], 8))
 
             if self.core.config["permission"]["change_dl"] and os.name != "nt":
                 try:
@@ -365,7 +370,7 @@ class Plugin(object):
                     gid = getgrnam(self.config["permission"]["group"])[2]
 
                     chown(location, uid, gid)
-                except Exception,e:
+                except Exception, e:
                     self.log.warning(_("Setting User and Group failed: %s") % str(e))
 
         name = self.pyfile.name
@@ -377,7 +382,9 @@ class Plugin(object):
 
         filename = save_join(location, name)
         try:
-            newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, chunks=self.getChunkCount(), resume=self.resumeDownload, progressNotify=self.pyfile.progress.setValue, disposition=disposition)
+            newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies,
+                                            chunks=self.getChunkCount(), resume=self.resumeDownload,
+                                            progressNotify=self.pyfile.progress.setValue, disposition=disposition)
         finally:
             self.pyfile.size = self.req.size
 
@@ -387,7 +394,7 @@ class Plugin(object):
             filename = save_join(location, newname)
 
         if self.core.config["permission"]["change_file"]:
-            chmod(filename, int(self.core.config["permission"]["file"],8))
+            chmod(filename, int(self.core.config["permission"]["file"], 8))
 
         if self.core.config["permission"]["change_dl"] and os.name != "nt":
             try:
@@ -395,20 +402,20 @@ class Plugin(object):
                 gid = getgrnam(self.config["permission"]["group"])[2]
 
                 chown(filename, uid, gid)
-            except Exception,e:
+            except Exception, e:
                 self.log.warning(_("Setting User and Group failed: %s") % str(e))
 
         self.lastDownload = filename
         return self.lastDownload
 
-    def checkDownload(self, rules, api_size=0 ,max_size=50000, delete=True, read_size=0):
+    def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0):
         """ checks the content of the last downloaded file
         rules - dict with names and rules to match(re or strings)
         size - excpected size
         @return name of first rule matched or None"""
 
         if not exists(self.lastDownload): return None
-        
+
         size = stat(self.lastDownload)
         size = size.st_size
 
@@ -441,9 +448,24 @@ class Plugin(object):
         return password
 
 
-    def checkForSameFiles(self):
+    def checkForSameFiles(self, starting=False):
         """ checks if same file was/is downloaded within same package and raise exception """
-        pass
+
+        pack = self.pyfile.package()
+
+        cache = self.core.files.cache.values()
+
+        for pyfile in cache:
+            if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
+                if pyfile.status in (0, 12): #finished or downloading
+                    raise SkipDownload(pyfile.pluginname)
+                elif pyfile.status in (5, 7) and starting: #a download is waiting and was appenrently started before
+                    raise SkipDownload(pyfile.pluginname)
+
+        #TODO check same packagenames
+        pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.packageid, self.pyfile.name)
+        if pyfile:
+            raise SkipDownload(pyfile[0])
 
     def clean(self):
         """ clean everything and remove references """
diff --git a/module/web/json_app.py b/module/web/json_app.py
index 9e9536a40..738f5ac63 100644
--- a/module/web/json_app.py
+++ b/module/web/json_app.py
@@ -142,6 +142,8 @@ def package(id):
                 pyfile["icon"] = "status_waiting.png"
             elif pyfile["status"] == 8:
                 pyfile["icon"] = "status_failed.png"
+            elif pyfile["status"] == 4:
+                pyfile["icon"] = "arrow_right.png"
             elif pyfile["status"] in (11, 13):
                 pyfile["icon"] = "status_proc.png"
             else:
diff --git a/module/web/media/default/img/arrow_right.png b/module/web/media/default/img/arrow_right.png
new file mode 100644
index 000000000..b1a181923
--- /dev/null
+++ b/module/web/media/default/img/arrow_right.png