online check for container

9 files changed, 223 insertions, 36 deletions

diff --git a/module/Api.py b/module/Api.py
index 221809dc2..0cc2dc1d4 100644
--- a/module/Api.py
+++ b/module/Api.py
@@ -28,8 +28,6 @@ from PyFile import PyFile
 from database.UserDatabase import ROLE
 from utils import freeSpace, compare_time
 from common.packagetools import parseNames
-from network.RequestFactory import getURL
-
 
 class Api(Iface):
     """
@@ -266,8 +264,6 @@ class Api(Iface):
         return pid
 
     def parseURLs(self, html):
-        html = getURL(html)
-
         # TODO parse
         urls = []
 
@@ -286,7 +282,7 @@ class Api(Iface):
 
         return plugins
 
-    def checkOnlineStatus(self, urls):
+    def checkOnlineStatus(self, urls, container=None):
         """ initiates online status check
 
         :param urls:
@@ -294,7 +290,7 @@ class Api(Iface):
         """
         data = self.core.pluginManager.parseUrls(urls)
 
-        rid = self.core.threadManager.createResultThread(data)
+        rid = self.core.threadManager.createResultThread(data, False, container)
 
         tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data]
         data = parseNames(tmp)
@@ -307,6 +303,20 @@ class Api(Iface):
 
         return OnlineCheck(rid, result)
 
+    def checkOnlineStatusContainer(self, urls, container, data):
+        """ checks online status of files and container file
+
+        :param urls: list of urls
+        :param container: container file name
+        :param data: file content
+        :return: online check
+        """
+        th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb")
+        th.write(str(data))
+        th.close()
+
+        return self.checkOnlineStatus(urls, th.name)
+
     def pollResults(self, rid):
         """ Polls the result available for ResultID
 
@@ -351,7 +361,7 @@ class Api(Iface):
         :param dest: `Destination`
         :return: None
         """
-        data = self.core.pluginManager.parseUrls(urls)
+        data = self.core.pluginManager.parseUrls(links)
         self.core.threadManager.createResultThread(data, True)
 
         
@@ -846,11 +856,7 @@ class Api(Iface):
         :param plugin: pluginname
         :return: dict of attr names mapped to value {"name": value}
         """
-        info = self.core.hookManager.getAllInfo()
-        if plugin in info:
-            return info[plugin]
-        else:
-            return {}
+        return self.core.hookManager.getInfo(plugin)
 
     def changePassword(self, user, oldpw, newpw):
         """ changes password for specific user """
diff --git a/module/HookManager.py b/module/HookManager.py
index c67acd34a..5876debe4 100644
--- a/module/HookManager.py
+++ b/module/HookManager.py
@@ -272,6 +272,14 @@ class HookManager:
         return info
 
 
+    def getInfo(self, plugin):
+        info = {}
+        if plugin in self.pluginMap and self.pluginMap[plugin].info:
+            info = dict([(x, str(y) if type(y) != basestring else y)
+                for x, y in self.pluginMap[plugin].info.iteritems()])
+
+        return info
+
     def addEvent(self, event, func):
         """Adds an event listener for event name"""
         if event in self.events:
diff --git a/module/PluginThread.py b/module/PluginThread.py
index 0a8151a25..b32361f4e 100644
--- a/module/PluginThread.py
+++ b/module/PluginThread.py
@@ -407,7 +407,7 @@ class HookThread(PluginThread):
 
 
 class InfoThread(PluginThread):
-    def __init__(self, manager, data, pid=-1, rid=-1, add=False):
+    def __init__(self, manager, data, pid=-1, rid=-1, add=False, container=None):
         """Constructor"""
         PluginThread.__init__(self, manager)
 
@@ -417,6 +417,7 @@ class InfoThread(PluginThread):
 
         self.rid = rid #result id
         self.add = add #add packages instead of return result
+        self.container = container #container file
 
         self.cache = [] #accumulated data
 
@@ -453,10 +454,9 @@ class InfoThread(PluginThread):
 
                     self.updateCache(pluginname, result)
 
+            packs = parseNames([(name, url) for name, x, y, url in self.cache])
 
-            packs = parseNames([(name, url) for name, x,y, url in self.cache])
-
-            self.m.core.log.debug("Fetched and generated %d packages" % len(packs))
+            self.m.log.debug("Fetched and generated %d packages" % len(packs))
 
             for k, v in packs:
                 self.m.core.api.addPackage(k, v)
@@ -466,6 +466,22 @@ class InfoThread(PluginThread):
 
         else: #post the results
 
+
+            if self.container:
+                #attach container content
+                try:
+                    data = self.decryptContainer()
+                except:
+                    print_exc()
+                    self.m.log.error("Could not decrypt container.")
+                    data = []
+
+                for url, plugin in data:
+                    if plugin in plugins:
+                        plugins[plugin].append(url)
+                    else:
+                        plugins[plugin] = [url]
+
             self.m.infoResults[self.rid] = {}
 
             for pluginname, urls in plugins.iteritems():
@@ -527,11 +543,11 @@ class InfoThread(PluginThread):
                     process.append(url)
 
             if result:
-                self.m.core.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname))
+                self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname))
                 cb(pluginname, result)
 
             if process:
-                self.m.core.log.debug("Run Info Fetching for %s" % pluginname)
+                self.m.log.debug("Run Info Fetching for %s" % pluginname)
                 for result in plugin.getInfo(process):
                     #result = [ .. (name, size, status, url) .. ]
                     if not type(result) == list: result = [result]
@@ -541,13 +557,41 @@ class InfoThread(PluginThread):
 
                     cb(pluginname, result)
 
-            self.m.core.log.debug("Finished Info Fetching for %s" % pluginname)
+            self.m.log.debug("Finished Info Fetching for %s" % pluginname)
         except Exception, e:
-            self.m.core.log.warning(_("Info Fetching for %(name)s failed | %(err)s") %
-                                    {"name": pluginname, "err": str(e)})
+            self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") %
+                               {"name": pluginname, "err": str(e)})
             if self.m.core.debug:
                 print_exc()
 
-            #TODO: generate default results
+            # generate default results
             if err:
-                pass
+                result = [(url, 0, 3, url) for url in urls]
+                cb(pluginname, result)
+
+
+    def decryptContainer(self):
+        url, plugin = self.m.core.pluginManager.parseUrls([self.container])[0]
+        # decrypt only container
+        if plugin in self.m.core.pluginManager.containerPlugins:
+
+            # dummy pyfile
+            pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1)
+
+            pyfile.initPlugin()
+
+            # little plugin lifecycle
+            pyfile.plugin.setup()
+            pyfile.plugin.loadToDisk()
+            pyfile.plugin.decrypt(pyfile)
+            pyfile.plugin.deleteTmp()
+
+            for pack in pyfile.plugin.packages:
+                pyfile.plugin.urls.extend(pack[1])
+
+            data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls)
+            pyfile.release()
+
+            return data
+
+        return []
diff --git a/module/PyFile.py b/module/PyFile.py
index 540933a93..9390c9cca 100644
--- a/module/PyFile.py
+++ b/module/PyFile.py
@@ -132,7 +132,9 @@ class PyFile(object):
     @lock
     def release(self):
         """sync and remove from cache"""
-        self.sync()
+        # file has valid package
+        if self.packageid > 0:
+            self.sync()
 
         if hasattr(self, "plugin") and self.plugin:
             self.plugin.clean()
diff --git a/module/ThreadManager.py b/module/ThreadManager.py
index ce5b7bd1d..159b495cd 100644
--- a/module/ThreadManager.py
+++ b/module/ThreadManager.py
@@ -88,14 +88,14 @@ class ThreadManager:
         PluginThread.InfoThread(self, data, pid)
 
     @lock
-    def createResultThread(self, data, add=False):
+    def createResultThread(self, data, add=False, container=None):
         """ creates a thread to fetch online status, returns result id """
         self.timestamp = time() + 5 * 60
 
         rid = self.resultIDs
         self.resultIDs += 1
 
-        PluginThread.InfoThread(self, data, rid=rid, add=add)
+        PluginThread.InfoThread(self, data, rid=rid, add=add, container=container)
 
         return rid
 
diff --git a/module/remote/thriftbackend/pyload.thrift b/module/remote/thriftbackend/pyload.thrift
index d73021725..ebe1890e3 100644
--- a/module/remote/thriftbackend/pyload.thrift
+++ b/module/remote/thriftbackend/pyload.thrift
@@ -221,6 +221,7 @@ service Pyload {
 
   // parses results and generates packages
   OnlineCheck checkOnlineStatus(1: LinkList urls),
+  OnlineCheck checkOnlineStatusContainer(1: LinkList urls, 2: string filename, 3: binary data)
 
   // poll results from previosly started online check
   OnlineCheck pollResults(1: ResultID rid),
diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote
index cdae7ac2a..8c2b368e1 100755
--- a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote
+++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote
@@ -43,6 +43,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print '   checkURLs(LinkList urls)'
   print '   parseURLs(string html)'
   print '  OnlineCheck checkOnlineStatus(LinkList urls)'
+  print '  OnlineCheck checkOnlineStatusContainer(LinkList urls, string filename, string data)'
   print '  OnlineCheck pollResults(ResultID rid)'
   print '   statusDownloads()'
   print '  PackageData getPackageData(PackageID pid)'
@@ -261,6 +262,12 @@ elif cmd == 'checkOnlineStatus':
     sys.exit(1)
   pp.pprint(client.checkOnlineStatus(eval(args[0]),))
 
+elif cmd == 'checkOnlineStatusContainer':
+  if len(args) != 3:
+    print 'checkOnlineStatusContainer requires 3 args'
+    sys.exit(1)
+  pp.pprint(client.checkOnlineStatusContainer(eval(args[0]),args[1],args[2],))
+
 elif cmd == 'pollResults':
   if len(args) != 1:
     print 'pollResults requires 1 args'
diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py
index 008c42a56..816c6e7d1 100644
--- a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py
+++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py
@@ -106,6 +106,15 @@ class Iface(object):
     """
     pass
 
+  def checkOnlineStatusContainer(self, urls, filename, data):
+    """
+    Parameters:
+     - urls
+     - filename
+     - data
+    """
+    pass
+
   def pollResults(self, rid):
     """
     Parameters:
@@ -956,6 +965,40 @@ class Client(Iface):
       return result.success
     raise TApplicationException(TApplicationException.MISSING_RESULT, "checkOnlineStatus failed: unknown result");
 
+  def checkOnlineStatusContainer(self, urls, filename, data):
+    """
+    Parameters:
+     - urls
+     - filename
+     - data
+    """
+    self.send_checkOnlineStatusContainer(urls, filename, data)
+    return self.recv_checkOnlineStatusContainer()
+
+  def send_checkOnlineStatusContainer(self, urls, filename, data):
+    self._oprot.writeMessageBegin('checkOnlineStatusContainer', TMessageType.CALL, self._seqid)
+    args = checkOnlineStatusContainer_args()
+    args.urls = urls
+    args.filename = filename
+    args.data = data
+    args.write(self._oprot)
+    self._oprot.writeMessageEnd()
+    self._oprot.trans.flush()
+
+  def recv_checkOnlineStatusContainer(self, ):
+    (fname, mtype, rseqid) = self._iprot.readMessageBegin()
+    if mtype == TMessageType.EXCEPTION:
+      x = TApplicationException()
+      x.read(self._iprot)
+      self._iprot.readMessageEnd()
+      raise x
+    result = checkOnlineStatusContainer_result()
+    result.read(self._iprot)
+    self._iprot.readMessageEnd()
+    if result.success is not None:
+      return result.success
+    raise TApplicationException(TApplicationException.MISSING_RESULT, "checkOnlineStatusContainer failed: unknown result");
+
   def pollResults(self, rid):
     """
     Parameters:
@@ -2367,6 +2410,7 @@ class Processor(Iface, TProcessor):
     self._processMap["checkURLs"] = Processor.process_checkURLs
     self._processMap["parseURLs"] = Processor.process_parseURLs
     self._processMap["checkOnlineStatus"] = Processor.process_checkOnlineStatus
+    self._processMap["checkOnlineStatusContainer"] = Processor.process_checkOnlineStatusContainer
     self._processMap["pollResults"] = Processor.process_pollResults
     self._processMap["statusDownloads"] = Processor.process_statusDownloads
     self._processMap["getPackageData"] = Processor.process_getPackageData
@@ -2651,6 +2695,17 @@ class Processor(Iface, TProcessor):
     oprot.writeMessageEnd()
     oprot.trans.flush()
 
+  def process_checkOnlineStatusContainer(self, seqid, iprot, oprot):
+    args = checkOnlineStatusContainer_args()
+    args.read(iprot)
+    iprot.readMessageEnd()
+    result = checkOnlineStatusContainer_result()
+    result.success = self._handler.checkOnlineStatusContainer(args.urls, args.filename, args.data)
+    oprot.writeMessageBegin("checkOnlineStatusContainer", TMessageType.REPLY, seqid)
+    result.write(oprot)
+    oprot.writeMessageEnd()
+    oprot.trans.flush()
+
   def process_pollResults(self, seqid, iprot, oprot):
     args = pollResults_args()
     args.read(iprot)
@@ -3784,6 +3839,51 @@ class checkOnlineStatus_result(TBase):
     self.success = success
 
 
+class checkOnlineStatusContainer_args(TBase):
+  """
+  Attributes:
+   - urls
+   - filename
+   - data
+  """
+
+  __slots__ = [ 
+    'urls',
+    'filename',
+    'data',
+   ]
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.LIST, 'urls', (TType.STRING,None), None, ), # 1
+    (2, TType.STRING, 'filename', None, None, ), # 2
+    (3, TType.STRING, 'data', None, None, ), # 3
+  )
+
+  def __init__(self, urls=None, filename=None, data=None,):
+    self.urls = urls
+    self.filename = filename
+    self.data = data
+
+
+class checkOnlineStatusContainer_result(TBase):
+  """
+  Attributes:
+   - success
+  """
+
+  __slots__ = [ 
+    'success',
+   ]
+
+  thrift_spec = (
+    (0, TType.STRUCT, 'success', (OnlineCheck, OnlineCheck.thrift_spec), None, ), # 0
+  )
+
+  def __init__(self, success=None,):
+    self.success = success
+
+
 class pollResults_args(TBase):
   """
   Attributes:
diff --git a/pyLoadCli.py b/pyLoadCli.py
index 9760c08b7..9c456d422 100755
--- a/pyLoadCli.py
+++ b/pyLoadCli.py
@@ -23,7 +23,7 @@ from getopt import GetoptError, getopt
 import gettext
 import os
 from os import _exit
-from os.path import join, exists, abspath
+from os.path import join, exists, abspath, basename
 import sys
 from sys import exit
 from threading import Thread, Lock
@@ -321,23 +321,28 @@ class Cli:
                 if exists(join(OWD, url)):
                     f = open(join(OWD, url), "rb")
                     links.extend([x.strip() for x in f.readlines() if x.strip()])
+                    f.close()
                 else:
                     links.append(url)
 
             print _("Checking %d links:") % len(links)
             print
             rid = client.checkOnlineStatus(links).rid
-            while True:
-                sleep(1)
-                result = client.pollResults(rid)
-                for url, status in result.data.iteritems():
-                    if status.status == 2: check = "Online"
-                    elif status.status == 1: check = "Offline"
-                    else: check = "Unknown"
+            self.printOnlineCheck(client, rid)
 
-                    print "%-30s: %-30s %-8s\t %s" % (url, status.name, formatSize(status.size), check)
 
-                if result.rid == -1: break
+        elif command == "check_container":
+            path = args[0]
+            if not exists(join(OWD, path)):
+                print _("File does not exists.")
+                return
+
+            f = open(join(OWD, path), "rb")
+            content = f.read()
+            f.close()
+
+            rid = client.checkOnlineStatusContainer([], basename(f.name), content).rid
+            self.printOnlineCheck(client, rid)
 
 
         elif command == "pause":
@@ -363,6 +368,19 @@ class Cli:
         else:
             print_commands()
 
+    def printOnlineCheck(self, client, rid):
+        while True:
+            sleep(1)
+            result = client.pollResults(rid)
+            for url, status in result.data.iteritems():
+                if status.status == 2: check = "Online"
+                elif status.status == 1: check = "Offline"
+                else: check = "Unknown"
+
+                print "%-30s: %-30s %-8s\t %s" % (url, status.name, formatSize(status.size), check)
+
+            if result.rid == -1: break
+
 
 class RefreshThread(Thread):
     def __init__(self, cli):
@@ -446,6 +464,7 @@ def print_commands():
                 ("restart_file <fid> <fid2>...", _("Restart files")),
                 ("restart_package <pid> <pid2>...", _("Restart packages")),
                 ("check <linklist|url> ...", _("Check online status")),
+                ("check_container path", _("Checks online status of a container file")),
                 ("pause", _("Pause the server")),
                 ("unpause", _("continue downloads")),
                 ("toggle", _("Toggle pause/unpause")),