new linkgrabber

4 files changed, 32 insertions, 90 deletions

diff --git a/pyload/api/CollectorApi.py b/pyload/api/CollectorApi.py
deleted file mode 100644
index 49340285e..000000000
--- a/pyload/api/CollectorApi.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from pyload.Api import Api, RequirePerm, Permission
-
-from ApiComponent import ApiComponent
-
-class CollectorApi(ApiComponent):
-    """ Link collector """
-
-    @RequirePerm(Permission.All)
-    def getCollector(self):
-        pass
-
-    @RequirePerm(Permission.Add)
-    def addToCollector(self, links):
-        pass
-
-    @RequirePerm(Permission.Add)
-    def addFromCollector(self, name, new_name):
-        pass
-
-    @RequirePerm(Permission.Delete)
-    def deleteCollPack(self, name):
-        pass
-
-    @RequirePerm(Permission.Add)
-    def renameCollPack(self, name, new_name):
-        pass
-
-    @RequirePerm(Permission.Delete)
-    def deleteCollLink(self, url):
-        pass
-
-
-if Api.extend(CollectorApi):
-    del CollectorApi
\ No newline at end of file
diff --git a/pyload/api/DownloadApi.py b/pyload/api/DownloadApi.py
index 0a01007b5..d855dd882 100644
--- a/pyload/api/DownloadApi.py
+++ b/pyload/api/DownloadApi.py
@@ -12,17 +12,6 @@ class DownloadApi(ApiComponent):
     """ Component to create, add, delete or modify downloads."""
 
     @RequirePerm(Permission.Add)
-    def generateAndAddPackages(self, links, paused=False):
-        """Generates and add packages
-
-        :param links: list of urls
-        :param paused: paused package
-        :return: list of package ids
-        """
-        return [self.addPackageP(name, urls, "", paused) for name, urls
-                in self.generatePackages(links).iteritems()]
-
-    @RequirePerm(Permission.Add)
     def createPackage(self, name, folder, root, password="", site="", comment="", paused=False):
         """Create a new package.
 
diff --git a/pyload/api/DownloadPreparingApi.py b/pyload/api/DownloadPreparingApi.py
index 0a47fe5ab..d74f0aee2 100644
--- a/pyload/api/DownloadPreparingApi.py
+++ b/pyload/api/DownloadPreparingApi.py
@@ -3,7 +3,9 @@
 
 from itertools import chain
 
-from pyload.Api import Api, RequirePerm, Permission, OnlineCheck, LinkStatus, urlmatcher
+from pyload.Api import Api, DownloadStatus as DS,\
+    RequirePerm, Permission, OnlineCheck, LinkStatus, urlmatcher
+from pyload.utils import uniqify
 from pyload.utils.fs import join
 from pyload.utils.packagetools import parseNames
 from pyload.network.RequestFactory import getURL
@@ -14,33 +16,13 @@ class DownloadPreparingApi(ApiComponent):
     """ All kind of methods to parse links or retrieve online status """
 
     @RequirePerm(Permission.Add)
-    def parseURLs(self, html=None, url=None):
-        """Parses html content or any arbitrary text for links and returns result of `checkURLs`
-
-        :param html: html source
-        :return:
-        """
-        urls = []
-
-        if html:
-            urls += [x[0] for x in urlmatcher.findall(html)]
-
-        if url:
-            page = getURL(url)
-            urls += [x[0] for x in urlmatcher.findall(page)]
-
-        # remove duplicates
-        return self.checkURLs(set(urls))
-
-
-    @RequirePerm(Permission.Add)
-    def checkURLs(self, urls):
+    def parseLinks(self, links):
         """ Gets urls and returns pluginname mapped to list of matching urls.
 
-        :param urls:
+        :param links:
         :return: {plugin: urls}
         """
-        data, crypter = self.core.pluginManager.parseUrls(urls)
+        data, crypter = self.core.pluginManager.parseUrls(links)
         plugins = {}
 
         for url, plugin in chain(data, crypter):
@@ -52,31 +34,23 @@ class DownloadPreparingApi(ApiComponent):
         return plugins
 
     @RequirePerm(Permission.Add)
-    def checkOnlineStatus(self, urls):
+    def checkLinks(self, links):
         """ initiates online status check, will also decrypt files.
 
         :param urls:
         :return: initial set of data as :class:`OnlineCheck` instance containing the result id
         """
-        data, crypter = self.core.pluginManager.parseUrls(urls)
+        hoster, crypter = self.core.pluginManager.parseUrls(links)
 
         # initial result does not contain the crypter links
-        tmp = [(url, (url, LinkStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data]
+        tmp = [(url, LinkStatus(url, url, pluginname, -1, DS.Queued)) for url, pluginname in hoster + crypter]
         data = parseNames(tmp)
-        result = {}
-
-        for k, v in data.iteritems():
-            for url, status in v:
-                status.packagename = k
-                result[url] = status
-
-        data.update(crypter) # hoster and crypter will be processed
-        rid = self.core.threadManager.createResultThread(data, False)
+        rid = self.core.threadManager.createResultThread(data)
 
-        return OnlineCheck(rid, result)
+        return OnlineCheck(rid, data)
 
     @RequirePerm(Permission.Add)
-    def checkOnlineStatusContainer(self, urls, container, data):
+    def checkContainer(self, filename, data):
         """ checks online status of urls and a submitted container file
 
         :param urls: list of urls
@@ -84,11 +58,27 @@ class DownloadPreparingApi(ApiComponent):
         :param data: file content
         :return: :class:`OnlineCheck`
         """
-        th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb")
+        th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb")
         th.write(str(data))
         th.close()
-        urls.append(th.name)
-        return self.checkOnlineStatus(urls)
+        return self.checkLinks([th.name])
+
+    @RequirePerm(Permission.Add)
+    def checkHTML(self, html, url):
+        """Parses html content or any arbitrary text for links and returns result of `checkURLs`
+
+        :param html: html source
+        :return:
+        """
+        urls = []
+        if html:
+            urls += [x[0] for x in urlmatcher.findall(html)]
+        if url:
+            page = getURL(url)
+            urls += [x[0] for x in urlmatcher.findall(page)]
+
+        # remove duplicates
+        return self.checkLinks(uniqify(urls))
 
     @RequirePerm(Permission.Add)
     def pollResults(self, rid):
diff --git a/pyload/api/__init__.py b/pyload/api/__init__.py
index 1348fd26f..bea46011d 100644
--- a/pyload/api/__init__.py
+++ b/pyload/api/__init__.py
@@ -1,5 +1,5 @@
 __all__ = ["CoreApi", "ConfigApi", "DownloadApi", "DownloadPreparingApi", "FileApi",
-           "CollectorApi", "UserInteractionApi", "AccountApi", "AddonApi"]
+            "UserInteractionApi", "AccountApi", "AddonApi"]
 
 # Import all components
 # from .import *