closed #364, changes to online check api

10 files changed, 100 insertions, 39 deletions

diff --git a/module/Api.py b/module/Api.py
index 1d8d4d77e..d36ecab33 100644
--- a/module/Api.py
+++ b/module/Api.py
@@ -281,17 +281,42 @@ class Api(Iface):
         return plugins
 
     def checkOnlineStatus(self, urls):
+        """ initiates online status check
+
+        :param urls:
+        :return: initial set of data and the result id
+        """
         data = self.core.pluginManager.parseUrls(urls)
-        return self.core.threadManager.createResultThread(data)
+
+        rid = self.core.threadManager.createResultThread(data)
+
+        tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data]
+        data = parseNames(tmp)
+        result = {}
+        
+        for k, v in data.iteritems():
+            for url, status in v:
+                status.packagename = k
+                result[url] = status
+
+        return OnlineCheck(rid, result)
 
     def pollResults(self, rid):
         """ Polls the result available for ResultID
-        :param rid:
+
+        :param rid: if -1 no more data is available
         :return:
         """
         self.core.threadManager.timestamp = time() + 5 * 60
 
-        return self.core.threadManager.getInfoResult(rid)
+        result = self.core.threadManager.getInfoResult(rid)
+
+        if "ALL_INFO_FETCHED" in result:
+            del result["ALL_INFO_FETCHED"]
+            return OnlineCheck(-1, result)
+        else:
+            return OnlineCheck(rid, result)
+
 
 
     def generatePackages(self, links):
@@ -620,7 +645,7 @@ class Api(Iface):
         if task:
             task.setWatingForUser(exclusive=exclusive)
             data, type, result = task.getCaptcha()
-            t = CaptchaTask(int(task.tid), standard_b64encode(data), type, result)
+            t = CaptchaTask(int(task.id), standard_b64encode(data), type, result)
             return t
         else:
             return CaptchaTask()
diff --git a/module/PluginThread.py b/module/PluginThread.py
index 5492f3ec4..a036b4878 100644
--- a/module/PluginThread.py
+++ b/module/PluginThread.py
@@ -455,12 +455,9 @@ class InfoThread(PluginThread):
 
                 else:
                     #generate default result
-                    tmp = [(url, (url, OnlineStatus(url, pluginname, 3, 0))) for url in urls]
-                    result = parseNames(tmp)
-                    for k in result.iterkeys():
-                        result[k] = dict(result[k])
+                    result = [(url, 0, 3, url) for url in urls]
 
-                    self.m.setInfoResults(self.rid, result)
+                    self.updateResult(pluginname, result, True)
 
             self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {}
 
@@ -478,12 +475,15 @@ class InfoThread(PluginThread):
 
         if len(self.cache) >= 20 or force:
             #used for package generating
-            tmp = [(name, (url, OnlineStatus(name, plugin, status, int(size))))
+            tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size))))
             for name, size, status, url in self.cache]
 
-            result = parseNames(tmp)
-            for k in result.iterkeys():
-                result[k] = dict(result[k])
+            data = parseNames(tmp)
+            result = {}
+            for k, v in data.iteritems():
+                for url, status in v:
+                    status.packagename = k
+                    result[url] = status
 
             self.m.setInfoResults(self.rid, result)
 
diff --git a/module/ThreadManager.py b/module/ThreadManager.py
index ba75764c5..07ab69548 100644
--- a/module/ThreadManager.py
+++ b/module/ThreadManager.py
@@ -112,11 +112,7 @@ class ThreadManager:
 
     @lock
     def setInfoResults(self, rid, result):
-        for k, v in result.iteritems():
-            if k in self.infoResults[rid]:
-                self.infoResults[rid][k].update(v)
-            else:
-                self.infoResults[rid][k] = v
+        self.infoResults[rid].update(result)
 
     def downloadingIds(self):
         """get a list of the currently downloading pyfile's ids"""
@@ -211,7 +207,7 @@ class ThreadManager:
 
     def getIP(self):
         """retrieve current ip"""
-        services = [("http://www.whatismyip.com/automation/n09230945.asp", "(\S+)"),
+        services = [("http://automation.whatismyip.com/n09230945.asp", "(\S+)"),
                     ("http://checkip.dyndns.org/",".*Current IP Address: (\S+)</body>.*")]
 
         ip = ""
diff --git a/module/common/packagetools.py b/module/common/packagetools.py
index 6b37c0198..9912a3f95 100644
--- a/module/common/packagetools.py
+++ b/module/common/packagetools.py
@@ -5,6 +5,8 @@
 import re
 from urlparse import urlparse
 
+from ..remote.thriftbackend.thriftgen.pyload.ttypes import OnlineStatus
+
 def matchFirst(string, *args):
     """ matches against list of regexp and returns first match"""
     for patternlist in args:
diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py
index 94c20cd5a..56b18b749 100644
--- a/module/plugins/PluginManager.py
+++ b/module/plugins/PluginManager.py
@@ -233,6 +233,10 @@ class PluginManager():
         if name in self.hosterPlugins:
             plugin = self.hosterPlugins[name]
 
+        if not plugin:
+            self.log.warning("Plugin %s not found." % name)
+            plugin = self.hosterPlugins["BasePlugin"]
+
         if "new_module" in plugin and not original:
             return plugin["new_module"]
 
diff --git a/module/remote/thriftbackend/pyload.thrift b/module/remote/thriftbackend/pyload.thrift
index 2995ce207..d73021725 100644
--- a/module/remote/thriftbackend/pyload.thrift
+++ b/module/remote/thriftbackend/pyload.thrift
@@ -160,10 +160,17 @@ struct ServiceCall {
 struct OnlineStatus {
     1: string name,
     2: PluginName plugin,
-    3: DownloadStatus status,
-    4: i64 size,   // size <= 0 : unknown
+    3: string packagename,
+    4: DownloadStatus status,
+    5: i64 size,   // size <= 0 : unknown
 }
 
+struct OnlineCheck {
+    1: ResultID rid, // -1 -> nothing more to get
+    2: map<string, OnlineStatus> data, //url to result
+}
+
+
 // exceptions
 
 exception PackageDoesNotExists{
@@ -213,9 +220,10 @@ service Pyload {
   map<PluginName, LinkList> parseURLs(1: string html),
 
   // parses results and generates packages
-  ResultID checkOnlineStatus(1: LinkList urls),
-  // poll results from previosly started online check , packagename - url - status
-  map<string, map<string, OnlineStatus>> pollResults(1: ResultID rid),
+  OnlineCheck checkOnlineStatus(1: LinkList urls),
+
+  // poll results from previosly started online check
+  OnlineCheck pollResults(1: ResultID rid),
 
   // downloads - information
   list<DownloadInfo> statusDownloads(),
diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote
index 0c0e70bd4..cdae7ac2a 100755
--- a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote
+++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote
@@ -42,8 +42,8 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help':
   print '   generatePackages(LinkList links)'
   print '   checkURLs(LinkList urls)'
   print '   parseURLs(string html)'
-  print '  ResultID checkOnlineStatus(LinkList urls)'
-  print '   pollResults(ResultID rid)'
+  print '  OnlineCheck checkOnlineStatus(LinkList urls)'
+  print '  OnlineCheck pollResults(ResultID rid)'
   print '   statusDownloads()'
   print '  PackageData getPackageData(PackageID pid)'
   print '  PackageData getPackageInfo(PackageID pid)'
diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py
index 009d850c2..008c42a56 100644
--- a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py
+++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py
@@ -3777,7 +3777,7 @@ class checkOnlineStatus_result(TBase):
    ]
 
   thrift_spec = (
-    (0, TType.I32, 'success', None, None, ), # 0
+    (0, TType.STRUCT, 'success', (OnlineCheck, OnlineCheck.thrift_spec), None, ), # 0
   )
 
   def __init__(self, success=None,):
@@ -3814,7 +3814,7 @@ class pollResults_result(TBase):
    ]
 
   thrift_spec = (
-    (0, TType.MAP, 'success', (TType.STRING,None,TType.MAP,(TType.STRING,None,TType.STRUCT,(OnlineStatus, OnlineStatus.thrift_spec))), None, ), # 0
+    (0, TType.STRUCT, 'success', (OnlineCheck, OnlineCheck.thrift_spec), None, ), # 0
   )
 
   def __init__(self, success=None,):
diff --git a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py
index f7052bc28..e5d22805e 100644
--- a/module/remote/thriftbackend/thriftgen/pyload/ttypes.py
+++ b/module/remote/thriftbackend/thriftgen/pyload/ttypes.py
@@ -606,6 +606,7 @@ class OnlineStatus(TBase):
   Attributes:
    - name
    - plugin
+   - packagename
    - status
    - size
   """
@@ -613,6 +614,7 @@ class OnlineStatus(TBase):
   __slots__ = [ 
     'name',
     'plugin',
+    'packagename',
     'status',
     'size',
    ]
@@ -621,17 +623,42 @@ class OnlineStatus(TBase):
     None, # 0
     (1, TType.STRING, 'name', None, None, ), # 1
     (2, TType.STRING, 'plugin', None, None, ), # 2
-    (3, TType.I32, 'status', None, None, ), # 3
-    (4, TType.I64, 'size', None, None, ), # 4
+    (3, TType.STRING, 'packagename', None, None, ), # 3
+    (4, TType.I32, 'status', None, None, ), # 4
+    (5, TType.I64, 'size', None, None, ), # 5
   )
 
-  def __init__(self, name=None, plugin=None, status=None, size=None,):
+  def __init__(self, name=None, plugin=None, packagename=None, status=None, size=None,):
     self.name = name
     self.plugin = plugin
+    self.packagename = packagename
     self.status = status
     self.size = size
 
 
+class OnlineCheck(TBase):
+  """
+  Attributes:
+   - rid
+   - data
+  """
+
+  __slots__ = [ 
+    'rid',
+    'data',
+   ]
+
+  thrift_spec = (
+    None, # 0
+    (1, TType.I32, 'rid', None, None, ), # 1
+    (2, TType.MAP, 'data', (TType.STRING,None,TType.STRUCT,(OnlineStatus, OnlineStatus.thrift_spec)), None, ), # 2
+  )
+
+  def __init__(self, rid=None, data=None,):
+    self.rid = rid
+    self.data = data
+
+
 class PackageDoesNotExists(TExceptionBase):
   """
   Attributes:
diff --git a/pyLoadCli.py b/pyLoadCli.py
index a59832c73..9760c08b7 100755
--- a/pyLoadCli.py
+++ b/pyLoadCli.py
@@ -326,19 +326,18 @@ class Cli:
 
             print _("Checking %d links:") % len(links)
             print
-            rid = client.checkOnlineStatus(links)
+            rid = client.checkOnlineStatus(links).rid
             while True:
                 sleep(1)
                 result = client.pollResults(rid)
-                for pack in result.itervalues():
-                    for url, status in pack.iteritems():
-                        if status.status == 2: check = "Online"
-                        elif status.status == 1: check = "Offline"
-                        else: check = "Unknown"
+                for url, status in result.data.iteritems():
+                    if status.status == 2: check = "Online"
+                    elif status.status == 1: check = "Offline"
+                    else: check = "Unknown"
 
-                        print "%-30s: %-30s %-8s\t %s" % (url, status.name, formatSize(status.size), check)
+                    print "%-30s: %-30s %-8s\t %s" % (url, status.name, formatSize(status.size), check)
 
-                if "ALL_INFO_FETCHED" in result: break
+                if result.rid == -1: break
 
 
         elif command == "pause":