started to integrate package generator

9 files changed, 475 insertions, 155 deletions

diff --git a/module/Api.py b/module/Api.py
index fd65dff92..dbdd2b04c 100644
--- a/module/Api.py
+++ b/module/Api.py
@@ -27,6 +27,7 @@ from remote.thriftbackend.thriftgen.pyload.Pyload import Iface
 from module.PyFile import PyFile
 from module.database.UserDatabase import ROLE
 from module.utils import freeSpace, compare_time
+from module.common.packagetools import parseNames
 
 
 class Api(Iface):
@@ -201,16 +202,6 @@ class Api(Iface):
         except:
             return ['No log available']
 
-    def parseURLs(self, html):
-        #TODO implement
-        pass
-
-    def checkURLs(self, urls):
-        pass
-
-    def checkOnlineStatus(self, urls):
-        pass
-
     def isTimeDownload(self):
         """Checks if pyload will start new downloads according to time in config .
 
@@ -273,6 +264,48 @@ class Api(Iface):
 
         return pid
 
+    def parseURLs(self, html):
+        #TODO implement
+        pass
+
+    def checkURLs(self, urls):
+        data = self.core.pluginManager.parseUrls(urls)
+        plugins = {}
+
+        for url, plugin in data:
+            if plugin in plugins:
+                plugins[plugin].append(url)
+            else:
+                plugins[plugin] = [url]
+
+        return plugins
+
+    def checkOnlineStatus(self, urls):
+        data = self.core.pluginManager.parseUrls(urls)
+        return self.core.threadManager.createResultThread(data)
+
+    def pollResults(self, rid):
+        pass
+
+    def generatePackages(self, links):
+        """ Parses links, generates packages names only from urls
+
+        :param links: list of urls
+        :return: package names mapt to urls
+        """
+        result = parseNames((x,x) for x in links)
+        return result
+
+    def generateAndAddPackages(self, links, dest=Destination.Queue):
+        """Generates and add packages
+
+        :param links: list of urls
+        :param dest: `Destination`
+        :return: list of package ids
+        """
+        return [self.addPackage(name, urls, dest) for name, urls
+                in self.generatePackages(links).iteritems()]
+
     def getPackageData(self, pid):
         """Returns complete information about package, and included files.
 
diff --git a/module/PluginThread.py b/module/PluginThread.py
index e0e3b17c9..a44981c52 100644
--- a/module/PluginThread.py
+++ b/module/PluginThread.py
@@ -30,8 +30,10 @@ from types import MethodType
 
 from pycurl import error
 
-from module.PyFile import PyFile
-from module.plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload
+from PyFile import PyFile
+from plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload
+from common.packagetools import parseNames
+from remote.thriftbackend.thriftgen.pyload.ttypes import OnlineStatus
 
 
 class PluginThread(Thread):
@@ -46,7 +48,8 @@ class PluginThread(Thread):
 
 
     def writeDebugReport(self, pyfile):
-        dump = "pyLoad %s Debug Report of %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % (self.m.core.api.getServerVersion(), pyfile.pluginname, format_exc())
+        dump = "pyLoad %s Debug Report of %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % (
+            self.m.core.api.getServerVersion(), pyfile.pluginname, format_exc())
 
         tb = exc_info()[2]
         stack = []
@@ -55,18 +58,17 @@ class PluginThread(Thread):
             tb = tb.tb_next
 
         for frame in stack[1:]:
-
             dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name,
-                                                     frame.f_code.co_filename,
-                                                     frame.f_lineno)
+                                                       frame.f_code.co_filename,
+                                                       frame.f_lineno)
 
             for key, value in frame.f_locals.items():
                 dump += "\t%20s = " % key
                 try:
                     dump += pformat(value) + "\n"
                 except Exception, e:
-                    dump += "<ERROR WHILE PRINTING VALUE> "+ str(e) +"\n"
-                    
+                    dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n"
+
             del frame
 
         del stack #delete it just to be sure...
@@ -80,7 +82,7 @@ class PluginThread(Thread):
                 try:
                     dump += pformat(attr) + "\n"
                 except Exception, e:
-                    dump += "<ERROR WHILE PRINTING VALUE> "+ str(e) +"\n"
+                    dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n"
 
         dump += "\nPYFILE OBJECT DUMP: \n\n"
 
@@ -91,14 +93,11 @@ class PluginThread(Thread):
                 try:
                     dump += pformat(attr) + "\n"
                 except Exception, e:
-                    dump += "<ERROR WHILE PRINTING VALUE> "+ str(e) +"\n"
-
+                    dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n"
 
         if pyfile.pluginname in self.m.core.config.plugin:
             dump += "\n\nCONFIG: \n\n"
-            dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) +"\n"
-
-
+            dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n"
 
         dump_name = "debug_%s_%s.txt" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S"))
         self.m.core.log.info("Debug Report written to %s" % dump_name)
@@ -142,7 +141,6 @@ class DownloadThread(PluginThread):
                 return True
 
             try:
-
                 if not pyfile.hasPlugin(): continue
                 #this pyfile was deleted while queueing
 
@@ -154,7 +152,6 @@ class DownloadThread(PluginThread):
                 pyfile.plugin.preprocessing(self)
 
             except NotImplementedError:
-
                 self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname)
                 pyfile.setStatus("failed")
                 pyfile.error = "Plugin does not work"
@@ -164,9 +161,9 @@ class DownloadThread(PluginThread):
             except Abort:
                 try:
                     self.m.log.info(_("Download aborted: %s") % pyfile.name)
-                except :
+                except:
                     pass
-                
+
                 pyfile.setStatus("aborted")
 
                 self.clean(pyfile)
@@ -182,14 +179,12 @@ class DownloadThread(PluginThread):
                 continue
 
             except Retry, e:
-
                 reason = e.args[0]
-                self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name" : pyfile.name, "msg": reason})
+                self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason})
                 self.queue.put(pyfile)
                 continue
 
             except Fail, e:
-
                 msg = e.args[0]
 
                 if msg == "offline":
@@ -218,7 +213,7 @@ class DownloadThread(PluginThread):
                 if code in (7, 18, 28, 52, 56):
                     self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry."))
                     wait = time() + 60
-                    
+
                     pyfile.waitUntil = wait
                     pyfile.setStatus("waiting")
                     while time() < wait:
@@ -247,10 +242,10 @@ class DownloadThread(PluginThread):
                 continue
 
             except SkipDownload, e:
-
                 pyfile.setStatus("skipped")
 
-                self.m.log.info(_("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message})
+                self.m.log.info(
+                    _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message})
 
                 self.clean(pyfile)
 
@@ -279,7 +274,6 @@ class DownloadThread(PluginThread):
                 self.m.core.files.checkAllLinksProcessed()
                 exc_clear()
 
-
             self.m.log.info(_("Download finished: %s") % pyfile.name)
             #pyfile.plugin.req.clean()
 
@@ -302,7 +296,6 @@ class DownloadThread(PluginThread):
         self.put("quit")
 
 
-
 class DecrypterThread(PluginThread):
     """thread for decrypting"""
 
@@ -330,12 +323,10 @@ class DecrypterThread(PluginThread):
             self.active.plugin.preprocessing(self)
 
         except NotImplementedError:
-
             self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname)
             return
 
         except Fail, e:
-
             msg = e.args[0]
 
             if msg == "offline":
@@ -343,28 +334,25 @@ class DecrypterThread(PluginThread):
                 self.m.log.warning(_("Download is offline: %s") % self.active.name)
             else:
                 self.active.setStatus("failed")
-                self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % { "name" : self.active.name, "msg":msg })
+                self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": msg})
                 self.active.error = msg
 
             return
-            
+
         except Abort:
-        
             self.m.log.info(_("Download aborted: %s") % pyfile.name)
             pyfile.setStatus("aborted")
-            
+
             return
 
         except Retry:
-            
             self.m.log.info(_("Retrying %s") % self.active.name)
             retry = True
             return self.run()
 
         except Exception, e:
-
             self.active.setStatus("failed")
-            self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % { "name" : self.active.name, "msg" :str(e) })
+            self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": str(e)})
             self.active.error = str(e)
 
             if self.m.core.debug:
@@ -391,6 +379,7 @@ class DecrypterThread(PluginThread):
         if not retry:
             pyfile.delete()
 
+
 class HookThread(PluginThread):
     """thread for hooks"""
 
@@ -403,7 +392,7 @@ class HookThread(PluginThread):
         self.active = pyfile
 
         m.localThreads.append(self)
-        
+
         if isinstance(pyfile, PyFile):
             pyfile.setStatus("processing")
 
@@ -411,22 +400,26 @@ class HookThread(PluginThread):
 
     def run(self):
         self.f(self.active)
-        
+
         self.m.localThreads.remove(self)
         if isinstance(self.active, PyFile):
             self.active.finishIfDone()
 
 
 class InfoThread(PluginThread):
-
     #----------------------------------------------------------------------
-    def __init__(self, manager, data, pid):
+    def __init__(self, manager, data, pid=-1, rid=-1):
         """Constructor"""
         PluginThread.__init__(self, manager)
 
         self.data = data
         self.pid = pid # package id
         # [ .. (name, plugin) .. ]
+
+        self.rid = rid #result id
+
+        self.cache = [] #accumulated data
+
         self.start()
 
     #----------------------------------------------------------------------
@@ -441,21 +434,88 @@ class InfoThread(PluginThread):
             else:
                 plugins[plugin] = [url]
 
-        for pluginname, urls in plugins.iteritems():
-            plugin = self.m.core.pluginManager.getPlugin(pluginname, True)
-            if hasattr(plugin, "getInfo"):
-                try:
-                    self.m.core.log.debug("Run Info Fetching for %s" % pluginname)
-                    for result in plugin.getInfo(urls):
-                        #result = [ .. (name, size, status, url) .. ]
-                        if not type(result) == list: result = [result]
-                        self.m.core.files.updateFileInfo(result, self.pid)
+        #directly write to database
+        if self.pid > -1:
+            for pluginname, urls in plugins.iteritems():
+                plugin = self.m.core.pluginManager.getPlugin(pluginname, True)
+                if hasattr(plugin, "getInfo"):
+                    self.fetchForPlugin(pluginname, plugin, urls, self.updateDB)
+                    self.m.core.files.save()
 
-                    self.m.core.log.debug("Finished Info Fetching for %s" % pluginname)
+        else: #post the results
 
-                    self.m.core.files.save()
-                except Exception, e:
-                    self.m.core.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % {"name": pluginname, "err": str(e)} )
-                    if self.m.core.debug:
-                        print_exc()
-                    
+            self.m.infoResults[self.rid] = {}
+
+            for pluginname, urls in plugins.iteritems():
+                plugin = self.m.core.pluginManager.getPlugin(pluginname, True)
+                if hasattr(plugin, "getInfo"):
+                    self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True)
+
+                    #force to process cache
+                    if self.cache:
+                        self.updateResult(pluginname, [], True)
+
+                else:
+                    #generate default result
+                    pass
+
+            self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = []
+
+
+    def updateDB(self, plugin, result):
+        self.m.core.files.updateFileInfo(result, self.pid)
+
+    def updateResult(self, plugin, result, force=False):
+        #parse package name and generate result
+        #accumulate results
+
+        self.cache.extend(result)
+
+        if len(self.cache) > 20 or force:
+            #used for package generating
+            tmp = [(name, (url, OnlineStatus(name, plugin, status, int(size))))
+            for name, size, status, url in self.cache]
+
+            result = parseNames(tmp)
+            for k in result.iterkeys():
+                result[k] = dict(result[k])
+
+            print result
+
+            self.cache = []
+
+    def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None):
+        try:
+            result = [] #result loaded from cache
+            process = [] #urls to process
+            for url in urls:
+                if url in self.m.infoCache:
+                    result.append(self.m.infoCache[url])
+                else:
+                    process.append(url)
+
+            if result:
+                self.m.core.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname))
+                cb(pluginname, result)
+
+            if process:
+                self.m.core.log.debug("Run Info Fetching for %s" % pluginname)
+                for result in plugin.getInfo(process):
+                    #result = [ .. (name, size, status, url) .. ]
+                    if not type(result) == list: result = [result]
+
+                    for res in result:
+                        self.m.infoCache[res[3]] = res
+
+                    cb(pluginname, result)
+
+            self.m.core.log.debug("Finished Info Fetching for %s" % pluginname)
+        except Exception, e:
+            self.m.core.log.warning(_("Info Fetching for %(name)s failed | %(err)s") %
+                                    {"name": pluginname, "err": str(e)})
+            if self.m.core.debug:
+                print_exc()
+
+            #TODO: generate default results
+            if err:
+                pass
diff --git a/module/ThreadManager.py b/module/ThreadManager.py
index 2c3d9af95..0ee59b427 100644
--- a/module/ThreadManager.py
+++ b/module/ThreadManager.py
@@ -33,11 +33,11 @@ from module.PyFile import PyFile
 from module.network.RequestFactory import getURL
 from module.utils import freeSpace
 
-########################################################################
+
 class ThreadManager:
     """manages the download threads, assign jobs, reconnect etc"""
 
-    #----------------------------------------------------------------------
+
     def __init__(self, core):
         """Constructor"""
         self.core = core
@@ -52,21 +52,31 @@ class ThreadManager:
         self.reconnecting.clear()
         self.downloaded = 0 #number of files downloaded since last cleanup
 
+        # some operations require to fetch url info from hoster, so we caching them so it wont be done twice
+        # contains a timestamp and will be purged after timeout
+        self.infoCache = {}
+
+        # pool of ids for online check
+        self.resultIDs = 0
+
+        # threads which are fetching hoster results
+        self.infoResults = {}
+        #timeout for cache purge
+        self.timestamp = 0
+
         pycurl.global_init(pycurl.GLOBAL_DEFAULT)
 
         for i in range(0, self.core.config.get("download", "max_downloads")):
             self.createThread()
 
 
-
-    #----------------------------------------------------------------------
     def createThread(self):
         """create a download thread"""
 
         thread = PluginThread.DownloadThread(self)
         self.threads.append(thread)
 
-    #----------------------------------------------------------------------
+
     def createInfoThread(self, data, pid):
         """
         start a thread whichs fetches online status and other infos
@@ -75,19 +85,28 @@ class ThreadManager:
 
         PluginThread.InfoThread(self, data, pid)
 
+    def createResultThread(self, data):
+        """ creates a thread to fetch online status, returns result id """
+
+        rid = self.resultIDs
+        self.resultIDs += 1
+
+        PluginThread.InfoThread(self, data, rid=rid)
+
+        return rid
+
 
-    #----------------------------------------------------------------------
     def downloadingIds(self):
         """get a list of the currently downloading pyfile's ids"""
         return [x.active.id for x in self.threads if x.active and isinstance(x.active, PyFile)]
 
-    #----------------------------------------------------------------------
+
     def processingIds(self):
         """get a id list of all pyfiles processed"""
         return [x.active.id for x in self.threads + self.localThreads if x.active and isinstance(x.active, PyFile)]
 
 
-    #----------------------------------------------------------------------
+    
     def work(self):
         """run all task which have to be done (this is for repetivive call by core)"""
         try:
diff --git a/module/common/packagetools.py b/module/common/packagetools.py
index 175c48937..4682b0dc1 100644
--- a/module/common/packagetools.py
+++ b/module/common/packagetools.py
@@ -5,98 +5,86 @@
 import re
 from urlparse import urlparse
 
+def matchFirst(string, *args):
+    """ matches against list of regexp and returns first match"""
+    for patternlist in args:
+        for pattern in patternlist:
+            r = pattern.search(string)
+            if r is not None:
+                name = r.group(1)
+                return name
+
+    return string
+
+
 def parseNames(files):
+    """ Generates packages names from name, data lists
+
+    :param files: list of (name, data)
+    :return: packagenames mapt to data lists (eg. urls)
+    """
     packs = {}
 
     endings = "\\.(3gp|7zip|7z|abr|ac3|aiff|aifc|aif|ai|au|avi|bin|bz2|cbr|cbz|ccf|cue|cvd|chm|dta|deb|divx|djvu|dlc|dmg|doc|docx|dot|eps|exe|ff|flv|f4v|gsd|gif|gz|iwd|iso|ipsw|java|jar|jpg|jpeg|jdeatme|load|mws|mw|m4v|m4a|mkv|mp2|mp3|mp4|mov|movie|mpeg|mpe|mpg|msi|msu|msp|nfo|npk|oga|ogg|ogv|otrkey|pkg|png|pdf|pptx|ppt|pps|ppz|pot|psd|qt|rmvb|rm|rar|ram|ra|rev|rnd|r\\d+|rpm|run|rsdf|rtf|sh(!?tml)|srt|snd|sfv|swf|tar|tif|tiff|ts|txt|viv|vivo|vob|wav|wmv|xla|xls|xpi|zeno|zip|z\\d+|_[_a-z]{2}|\\d+$)"
 
-    pat0 = re.compile("(.*)(\\.|_|-)pa?r?t?\\.?[0-9]+.(rar|exe)$", re.I)
-    pat1 = re.compile("(.*)(\\.|_|-)part\\.?[0]*[1].(rar|exe)$", re.I)
-    pat3 = re.compile("(.*)\\.rar$", re.I)
-    pat4 = re.compile("(.*)\\.r\\d+$", re.I)
-    pat5 = re.compile("(.*)(\\.|_|-)\\d+$", re.I)
-    rarPats = [ pat0, pat1, pat3, pat4, pat5 ]
+    rarPats = [re.compile("(.*)(\\.|_|-)pa?r?t?\\.?[0-9]+.(rar|exe)$", re.I),
+               re.compile("(.*)(\\.|_|-)part\\.?[0]*[1].(rar|exe)$", re.I),
+               re.compile("(.*)\\.rar$", re.I),
+               re.compile("(.*)\\.r\\d+$", re.I),
+               re.compile("(.*)(\\.|_|-)\\d+$", re.I)]
 
-    pat6 = re.compile("(.*)\\.zip$", re.I)
-    pat7 = re.compile("(.*)\\.z\\d+$", re.I)
-    pat8 = re.compile("(?is).*\\.7z\\.[\\d]+$", re.I)
-    pat9 = re.compile("(.*)\\.a.$", re.I)
-    zipPats = [ pat6, pat7, pat8, pat9 ]
+    zipPats = [re.compile("(.*)\\.zip$", re.I),
+               re.compile("(.*)\\.z\\d+$", re.I),
+               re.compile("(?is).*\\.7z\\.[\\d]+$", re.I),
+               re.compile("(.*)\\.a.$", re.I)]
 
-    pat10 = re.compile("(.*)\\._((_[a-z])|([a-z]{2}))(\\.|$)")
-    pat11 = re.compile("(.*)(\\.|_|-)[\\d]+(" + endings + "$)", re.I)
-    ffsjPats = [ pat10, pat11 ]
+    ffsjPats = [re.compile("(.*)\\._((_[a-z])|([a-z]{2}))(\\.|$)"),
+                re.compile("(.*)(\\.|_|-)[\\d]+(" + endings + "$)", re.I)]
 
-    pat12 = re.compile("(\\.?CD\\d+)", re.I)
-    pat13 = re.compile("(\\.?part\\d+)", re.I)
+    iszPats = [re.compile("(.*)\\.isz$", re.I),
+               re.compile("(.*)\\.i\\d{2}$", re.I)]
 
-    pat14 = re.compile("(.+)[\\.\\-_]+$")
+    pat1 = re.compile("(\\.?CD\\d+)", re.I)
+    pat2 = re.compile("(\\.?part\\d+)", re.I)
 
-    pat17 = re.compile("(.+)\\.\\d+\\.xtm$")
+    pat3 = re.compile("(.+)[\\.\\-_]+$")
+    pat4 = re.compile("(.+)\\.\\d+\\.xtm$")
 
-    pat18 = re.compile("(.*)\\.isz$", re.I)
-    pat19 = re.compile("(.*)\\.i\\d{2}$", re.I)
-    iszPats = [ pat18, pat19 ]
 
-    for file in files:
+    for file, url in files:
         # remove trailing /
         name = file.rstrip('/')
+
         # extract last path part .. if there is a path
         split = name.rsplit("/", 1)
         if len(split) > 1:
             name = split.pop(1)
 
+            #check if a already existing package may be ok for this file
+        #        found = False
+        #        for pack in packs:
+        #            if pack in file:
+        #                packs[pack].append(url)
+        #                found = True
+        #                break
+        #
+        #        if found: continue
 
-        #check if a already existing package may be ok for this file
-        found = False
-        for name in packs:
-            if name in file:
-                packs[name].append(file)
-                found = True
-                break
-
-        if found: continue
-
-
-        # unrar pattern
-        for pattern in rarPats:
-            r = pattern.search(name)
-            if r is not None:
-                name = r.group(1)
-                break
-
-        # 7zip/zip and hjmerge pattern
-        for pattern in zipPats:
-            r = pattern.search(name)
-            if r is not None:
-                name = r.group(1)
-                break
-
-        # isz pattern
-        for pattern in iszPats:
-            r = pattern.search(name)
-            if r is not None:
-                name = r.group(1)
-                break
+        # unrar pattern, 7zip/zip and hjmerge pattern, isz pattern, FFSJ pattern
+        name = matchFirst(name, rarPats, zipPats, iszPats, ffsjPats)
 
         # xtremsplit pattern
-        r = pat17.search(name)
+        r = pat4.search(name)
         if r is not None:
             name = r.group(1)
 
-        # FFSJ pattern
-        for pattern in ffsjPats:
-            r = pattern.search(name)
-            if r is not None:
-                name = r.group(1)
-                break
 
         # remove part and cd pattern
-        r = pat12.search(name)
+        r = pat1.search(name)
         if r is not None:
             name = name.replace(r.group(0), "")
 
-        r = pat13.search(name)
+        r = pat2.search(name)
         if r is not None:
             name = name.replace(r.group(0), "")
 
@@ -110,7 +98,7 @@ def parseNames(files):
                 name = name[:-length]
 
         # remove endings like . _ -
-        r = pat14.search(name)
+        r = pat3.search(name)
         if r is not None:
             name = r.group(1)
 
@@ -121,10 +109,10 @@ def parseNames(files):
         name = name.strip()
 
         # checks if name could be a hash
-        if file.find("file/"+name) >= 0:
+        if file.find("file/" + name) >= 0:
             name = ""
 
-        if file.find("files/"+name) >= 0:
+        if file.find("files/" + name) >= 0:
             name = ""
 
         r = re.search("^[0-9]+$", name, re.I)
@@ -139,8 +127,9 @@ def parseNames(files):
                 name = ""
 
         # fallback: package by hoster
-        if not len(name):
+        if not name:
             name = urlparse(file).hostname
+            if name: name = name.replace("ww.", "")
 
         # fallback : default name
         if not name:
@@ -148,8 +137,26 @@ def parseNames(files):
 
         # build mapping
         if name in packs:
-            packs[name].append(file)
+            packs[name].append(url)
         else:
-            packs[name] = [file]
-            
-    return packs
\ No newline at end of file
+            packs[name] = [url]
+
+    return packs
+
+
+if __name__ == "__main__":
+
+    from os.path import join
+    from pprint import pprint
+
+    f = open(join("..", "..", "testlinks2.txt"), "rb")
+    urls = [(x.strip(), x.strip()) for x in f.readlines() if x.strip()]
+    f.close()
+
+    print "Having %d urls." % len(urls)
+
+    packs = parseNames(urls)
+
+    pprint(packs)
+
+    print "Got %d urls." % sum([len(x) for x in packs.itervalues()])
\ No newline at end of file
diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py
index 4b55f0357..71c61942f 100644
--- a/module/plugins/hoster/BasePlugin.py
+++ b/module/plugins/hoster/BasePlugin.py
@@ -3,12 +3,13 @@
 
 import re
 from module.plugins.Hoster import Hoster
+from module.utils import html_unescape
 
 class BasePlugin(Hoster):
     __name__ = "BasePlugin"
     __type__ = "hoster"
     __pattern__ = r"^unmatchable$"
-    __version__ = "0.1"
+    __version__ = "0.11"
     __description__ = """Base Plugin when any other didnt fit"""
     __author_name__ = ("RaNaN")
     __author_mail__ = ("RaNaN@pyload.org")
@@ -38,7 +39,7 @@ class BasePlugin(Hoster):
 #        return
         if pyfile.url.startswith("http"):
 
-            pyfile.name = re.findall("([^/=]+)", pyfile.url)[-1]
+            pyfile.name = html_unescape(re.findall("([^/=]+)", pyfile.url)[-1])
             self.download(pyfile.url, disposition=True)
             
         else:
diff --git a/module/remote/thriftbackend/pyload.thrift b/module/remote/thriftbackend/pyload.thrift
index eff697d05..2995ce207 100644
--- a/module/remote/thriftbackend/pyload.thrift
+++ b/module/remote/thriftbackend/pyload.thrift
@@ -158,8 +158,8 @@ struct ServiceCall {
 }
 
 struct OnlineStatus {
-    1: string url,
-    2: string name,
+    1: string name,
+    2: PluginName plugin,
     3: DownloadStatus status,
     4: i64 size,   // size <= 0 : unknown
 }
@@ -206,10 +206,16 @@ service Pyload {
   bool toggleReconnect(),
 
   // download preparing
+
+  // packagename - urls
+  map<string, LinkList> generatePackages(1: LinkList links),
   map<PluginName, LinkList> checkURLs(1: LinkList urls),
   map<PluginName, LinkList> parseURLs(1: string html),
+
+  // parses results and generates packages
   ResultID checkOnlineStatus(1: LinkList urls),
-  map<PluginName, list<OnlineStatus>> pollResults(1: ResultID rid),
+  // poll results from previosly started online check , packagename - url - status
+  map<string, map<string, OnlineStatus>> pollResults(1: ResultID rid),
 
   // downloads - information
   list<DownloadInfo> statusDownloads(),
@@ -224,6 +230,7 @@ service Pyload {
   map<i16, FileID> getFileOrder(1: PackageID pid)
 
   // downloads - adding/deleting
+  list<PackageID> generateAndAddPackages(1: LinkList links, 2: Destination dest),
   PackageID addPackage(1: string name, 2: LinkList links, 3: Destination dest),
   void addFiles(1: PackageID pid, 2: LinkList links),
   void uploadContainer(1: string filename, 2: binary data),
diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote
index 2b055321b..0c0e70bd4 100755
--- a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote
+++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote
@@ -39,6 +39,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print '  bool isTimeDownload()'
   print '  bool isTimeReconnect()'
   print '  bool toggleReconnect()'
+  print '   generatePackages(LinkList links)'
   print '   checkURLs(LinkList urls)'
   print '   parseURLs(string html)'
   print '  ResultID checkOnlineStatus(LinkList urls)'
@@ -53,6 +54,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print '   getCollectorData()'
   print '   getPackageOrder(Destination destination)'
   print '   getFileOrder(PackageID pid)'
+  print '   generateAndAddPackages(LinkList links, Destination dest)'
   print '  PackageID addPackage(string name, LinkList links, Destination dest)'
   print '  void addFiles(PackageID pid, LinkList links)'
   print '  void uploadContainer(string filename, string data)'
@@ -235,6 +237,12 @@ elif cmd == 'toggleReconnect':
     sys.exit(1)
   pp.pprint(client.toggleReconnect())
 
+elif cmd == 'generatePackages':
+  if len(args) != 1:
+    print 'generatePackages requires 1 args'
+    sys.exit(1)
+  pp.pprint(client.generatePackages(eval(args[0]),))
+
 elif cmd == 'checkURLs':
   if len(args) != 1:
     print 'checkURLs requires 1 args'
@@ -319,6 +327,12 @@ elif cmd == 'getFileOrder':
     sys.exit(1)
   pp.pprint(client.getFileOrder(eval(args[0]),))
 
+elif cmd == 'generateAndAddPackages':
+  if len(args) != 2:
+    print 'generateAndAddPackages requires 2 args'
+    sys.exit(1)
+  pp.pprint(client.generateAndAddPackages(eval(args[0]),eval(args[1]),))
+
 elif cmd == 'addPackage':
   if len(args) != 3:
     print 'addPackage requires 3 args'
diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py
index de3611b6d..009d850c2 100644
--- a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py
+++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py
@@ -78,6 +78,13 @@ class Iface(object):
   def toggleReconnect(self, ):
     pass
 
+  def generatePackages(self, links):
+    """
+    Parameters:
+     - links
+    """
+    pass
+
   def checkURLs(self, urls):
     """
     Parameters:
@@ -156,6 +163,14 @@ class Iface(object):
     """
     pass
 
+  def generateAndAddPackages(self, links, dest):
+    """
+    Parameters:
+     - links
+     - dest
+    """
+    pass
+
   def addPackage(self, name, links, dest):
     """
     Parameters:
@@ -821,6 +836,36 @@ class Client(Iface):
       return result.success
     raise TApplicationException(TApplicationException.MISSING_RESULT, "toggleReconnect failed: unknown result");
 
+  def generatePackages(self, links):
+    """
+    Parameters:
+     - links
+    """
+    self.send_generatePackages(links)
+    return self.recv_generatePackages()
+
+  def send_generatePackages(self, links):
+    self._oprot.writeMessageBegin('generatePackages', TMessageType.CALL, self._seqid)
+    args = generatePackages_args()
+    args.links = links
+    args.write(self._oprot)
+    self._oprot.writeMessageEnd()
+    self._oprot.trans.flush()
+
+  def recv_generatePackages(self, ):
+    (fname, mtype, rseqid) = self._iprot.readMessageBegin()
+    if mtype == TMessageType.EXCEPTION:
+      x = TApplicationException()
+      x.read(self._iprot)
+      self._iprot.readMessageEnd()
+      raise x
+    result = generatePackages_result()
+    result.read(self._iprot)
+    self._iprot.readMessageEnd()
+    if result.success is not None:
+      return result.success
+    raise TApplicationException(TApplicationException.MISSING_RESULT, "generatePackages failed: unknown result");
+
   def checkURLs(self, urls):
     """
     Parameters:
@@ -1222,6 +1267,38 @@ class Client(Iface):
       return result.success
     raise TApplicationException(TApplicationException.MISSING_RESULT, "getFileOrder failed: unknown result");
 
+  def generateAndAddPackages(self, links, dest):
+    """
+    Parameters:
+     - links
+     - dest
+    """
+    self.send_generateAndAddPackages(links, dest)
+    return self.recv_generateAndAddPackages()
+
+  def send_generateAndAddPackages(self, links, dest):
+    self._oprot.writeMessageBegin('generateAndAddPackages', TMessageType.CALL, self._seqid)
+    args = generateAndAddPackages_args()
+    args.links = links
+    args.dest = dest
+    args.write(self._oprot)
+    self._oprot.writeMessageEnd()
+    self._oprot.trans.flush()
+
+  def recv_generateAndAddPackages(self, ):
+    (fname, mtype, rseqid) = self._iprot.readMessageBegin()
+    if mtype == TMessageType.EXCEPTION:
+      x = TApplicationException()
+      x.read(self._iprot)
+      self._iprot.readMessageEnd()
+      raise x
+    result = generateAndAddPackages_result()
+    result.read(self._iprot)
+    self._iprot.readMessageEnd()
+    if result.success is not None:
+      return result.success
+    raise TApplicationException(TApplicationException.MISSING_RESULT, "generateAndAddPackages failed: unknown result");
+
   def addPackage(self, name, links, dest):
     """
     Parameters:
@@ -2286,6 +2363,7 @@ class Processor(Iface, TProcessor):
     self._processMap["isTimeDownload"] = Processor.process_isTimeDownload
     self._processMap["isTimeReconnect"] = Processor.process_isTimeReconnect
     self._processMap["toggleReconnect"] = Processor.process_toggleReconnect
+    self._processMap["generatePackages"] = Processor.process_generatePackages
     self._processMap["checkURLs"] = Processor.process_checkURLs
     self._processMap["parseURLs"] = Processor.process_parseURLs
     self._processMap["checkOnlineStatus"] = Processor.process_checkOnlineStatus
@@ -2300,6 +2378,7 @@ class Processor(Iface, TProcessor):
     self._processMap["getCollectorData"] = Processor.process_getCollectorData
     self._processMap["getPackageOrder"] = Processor.process_getPackageOrder
     self._processMap["getFileOrder"] = Processor.process_getFileOrder
+    self._processMap["generateAndAddPackages"] = Processor.process_generateAndAddPackages
     self._processMap["addPackage"] = Processor.process_addPackage
     self._processMap["addFiles"] = Processor.process_addFiles
     self._processMap["uploadContainer"] = Processor.process_uploadContainer
@@ -2528,6 +2607,17 @@ class Processor(Iface, TProcessor):
     oprot.writeMessageEnd()
     oprot.trans.flush()
 
+  def process_generatePackages(self, seqid, iprot, oprot):
+    args = generatePackages_args()
+    args.read(iprot)
+    iprot.readMessageEnd()
+    result = generatePackages_result()
+    result.success = self._handler.generatePackages(args.links)
+    oprot.writeMessageBegin("generatePackages", TMessageType.REPLY, seqid)
+    result.write(oprot)
+    oprot.writeMessageEnd()
+    oprot.trans.flush()
+
   def process_checkURLs(self, seqid, iprot, oprot):
     args = checkURLs_args()
     args.read(iprot)
@@ -2691,6 +2781,17 @@ class Processor(Iface, TProcessor):
     oprot.writeMessageEnd()
     oprot.trans.flush()
 
+  def process_generateAndAddPackages(self, seqid, iprot, oprot):
+    args = generateAndAddPackages_args()
+    args.read(iprot)
+    iprot.readMessageEnd()
+    result = generateAndAddPackages_result()
+    result.success = self._handler.generateAndAddPackages(args.links, args.dest)
+    oprot.writeMessageBegin("generateAndAddPackages", TMessageType.REPLY, seqid)
+    result.write(oprot)
+    oprot.writeMessageEnd()
+    oprot.trans.flush()
+
   def process_addPackage(self, seqid, iprot, oprot):
     args = addPackage_args()
     args.read(iprot)
@@ -3535,6 +3636,43 @@ class toggleReconnect_result(TBase):
     self.success = success
 
 
+class generatePackages_args(TBase):
+  """
+  Attributes:
+   - links
+  """
+
+  __slots__ = [ 
+    'links',
+   ]
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.LIST, 'links', (TType.STRING,None), None, ), # 1
+  )
+
+  def __init__(self, links=None,):
+    self.links = links
+
+
+class generatePackages_result(TBase):
+  """
+  Attributes:
+   - success
+  """
+
+  __slots__ = [ 
+    'success',
+   ]
+
+  thrift_spec = (
+    (0, TType.MAP, 'success', (TType.STRING,None,TType.LIST,(TType.STRING,None)), None, ), # 0
+  )
+
+  def __init__(self, success=None,):
+    self.success = success
+
+
 class checkURLs_args(TBase):
   """
   Attributes:
@@ -3676,7 +3814,7 @@ class pollResults_result(TBase):
    ]
 
   thrift_spec = (
-    (0, TType.MAP, 'success', (TType.STRING,None,TType.LIST,(TType.STRUCT,(OnlineStatus, OnlineStatus.thrift_spec))), None, ), # 0
+    (0, TType.MAP, 'success', (TType.STRING,None,TType.MAP,(TType.STRING,None,TType.STRUCT,(OnlineStatus, OnlineStatus.thrift_spec))), None, ), # 0
   )
 
   def __init__(self, success=None,):
@@ -4015,6 +4153,47 @@ class getFileOrder_result(TBase):
     self.success = success
 
 
+class generateAndAddPackages_args(TBase):
+  """
+  Attributes:
+   - links
+   - dest
+  """
+
+  __slots__ = [ 
+    'links',
+    'dest',
+   ]
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.LIST, 'links', (TType.STRING,None), None, ), # 1
+    (2, TType.I32, 'dest', None, None, ), # 2
+  )
+
+  def __init__(self, links=None, dest=None,):
+    self.links = links
+    self.dest = dest
+
+
+class generateAndAddPackages_result(TBase):
+  """
+  Attributes:
+   - success
+  """
+
+  __slots__ = [ 
+    'success',
+   ]
+
+  thrift_spec = (
+    (0, TType.LIST, 'success', (TType.I32,None), None, ), # 0
+  )
+
+  def __init__(self, success=None,):
+    self.success = success
+
+
 class addPackage_args(TBase):
   """
   Attributes:
diff --git a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py
index fcff55ed8..f7052bc28 100644
--- a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py
+++ b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py
@@ -604,30 +604,30 @@ class ServiceCall(TBase):
 class OnlineStatus(TBase):
   """
   Attributes:
-   - url
    - name
+   - plugin
    - status
    - size
   """
 
   __slots__ = [ 
-    'url',
     'name',
+    'plugin',
     'status',
     'size',
    ]
 
   thrift_spec = (
     None, # 0
-    (1, TType.STRING, 'url', None, None, ), # 1
-    (2, TType.STRING, 'name', None, None, ), # 2
+    (1, TType.STRING, 'name', None, None, ), # 1
+    (2, TType.STRING, 'plugin', None, None, ), # 2
     (3, TType.I32, 'status', None, None, ), # 3
     (4, TType.I64, 'size', None, None, ), # 4
   )
 
-  def __init__(self, url=None, name=None, status=None, size=None,):
-    self.url = url
+  def __init__(self, name=None, plugin=None, status=None, size=None,):
     self.name = name
+    self.plugin = plugin
     self.status = status
     self.size = size