complete new file_list, cleaned up

6 files changed, 257 insertions, 316 deletions

diff --git a/module/file_list.py b/module/file_list.py
index 1701b801b..6e1984704 100644
--- a/module/file_list.py
+++ b/module/file_list.py
@@ -1,159 +1,274 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-#
-#Copyright (C) 2009 RaNaN
-#
-#This program is free software; you can redistribute it and/or modify
-#it under the terms of the GNU General Public License as published by
-#the Free Software Foundation; either version 3 of the License,
-#or (at your option) any later version.
-#
-#This program is distributed in the hope that it will be useful,
-#but WITHOUT ANY WARRANTY; without even the implied warranty of
-#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-#See the GNU General Public License for more details.
-#
-#You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>.
-#
-###
 
-LIST_VERSION = 3
+"""
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License,
+    or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, see <http://www.gnu.org/licenses/>.
+    
+    @author: mkaay
+    @author: RaNaN
+    @version: v0.3
+    @list-version: v4
+"""
+
+LIST_VERSION = 4
 
 from threading import RLock
 from download_thread import Status
 import cPickle
 import re
-from module.remote.RequestObject import RequestObject
 import module.Plugin
 
+class NoSuchElementException(Exception):
+    pass
+
 class File_List(object):
     def __init__(self, core):
         self.core = core
-        self.files = []
-        self.data = {'version': LIST_VERSION, 'order': []}
         self.lock = RLock()
-        self.load()
-
-    def new_pyfile(self, url, folder):
-        url  = url.replace("\n", "")
-        pyfile = PyLoadFile(self.core, url)
-        pyfile.download_folder = self.core.config['general']['download_folder']
-        pyfile.id = self.get_id()
-        pyfile.folder = folder
-
-        return pyfile
-
-    def append(self, url, folder=""):
-        if not url:
-            return False
-        #@TODO: filter non existence and invalid links
-        #re.compile("https?://[-a-z0-9\.]{4,}(?::\d+)?/[^#?]+(?:#\S+)?",re.IGNORECASE)
-        new_file = self.new_pyfile(url, folder)
-        self.files.append(new_file)
-        self.data[new_file.id] = Data(url, folder)
-        self.data['order'].append(int(new_file.id))
-
-    def extend(self, urls):
-        for url in urls:
-            self.append(url)
-
-    def remove(self, pyfile):
-        if not self.core.config['general']['debug_mode']:
-            if pyfile in self.files:
-                self.files.remove(pyfile)
-
-            self.data['order'].remove(pyfile.id)
-            del self.data[pyfile.id]
-
-    def remove_id(self, pyid):
-        #also abort download
-        pyid = int(pyid)
-        found = False
-        for pyfile in self.files:
-            if pyfile.id == pyid:
-                self.files.remove(pyfile)
-                found = True
-                break
-
-        if not found:
-            for pyfile in self.core.thread_list.py_downloading:
-                if pyfile.id == pyid:
-                    pyfile.plugin.req.abort = True
-                    break
-            return False
-
-        self.data['order'].remove(pyid)
-        del self.data[pyid]
-
-    def get_id(self):
-        """return a free id"""
-        id = 1
-        while id in self.data.keys():
-            id += 1
-
-        return id
-
-    def move(self, id, offset=-1):
-        
-        for pyfile in self.files:
-            if pyfile.id == id:
-                index = self.files.index(pyfile)
-                pyfile = self.files.pop(index)
-                self.files.insert(index + offset, pyfile)
-                break
-
-        index = self.data['order'].index(id)
-        pyfile = self.data['order'].pop(index)
-        self.data['order'].insert(index + offset, pyfile)
-
-    def save(self):
-        self.lock.acquire()
-
-        output = open('links.pkl', 'wb')
-        cPickle.dump(self.data, output, -1)
-
-        #self.inform_client()
+        self.download_folder = self.core.config['general']['download_folder']
+        self.collector = self.pyLoadCollector(self)
+        self.packager = self.pyLoadPackager(self)
+        
+        self.data = {
+            "version": LIST_VERSION,
+            "queue": [],
+            "packages": [],
+            "collector": []
+        }
         
-        self.lock.release()
-
     def load(self):
+        self.lock.acquire()
         try:
             pkl_file = open('links.pkl', 'rb')
             obj = cPickle.load(pkl_file)
         except:
-            obj = {'version': LIST_VERSION, 'order': []}
-
-        if obj['version'] < LIST_VERSION:
-            obj = {'version': LIST_VERSION, 'order': []}
-
-        for i in obj['order']:
-            self.append(obj[i].url, obj[i].folder)
-
-        self.core.logger.info("Links loaded: " + str(int(len(obj) - 2)))
-
-    def inform_client(self):
-        obj = RequestObject()
-        obj.command = "file_list"
-        obj.data = self.data
+            obj = False
+        if obj['version'] == LIST_VERSION and obj:
+            self.data = obj
+        self.lock.release()
+        
+        if len(self.data["collector"]) > 0:
+            self.core.logger.info("Found %s links in linkcollector" % len(self.data["collector"]))
+        if len(self.data["packages"]) > 0:
+            self.core.logger.info("Found %s unqueued packages" % len(self.data["packages"]))
+        if len(self.data["queue"]) > 0:
+            self.core.logger.info("Added %s packages to queue" % len(self.data["queue"]))
+    
+    def save(self):
+        self.lock.acquire()
 
-        self.core.server.push_all(obj)
+        output = open('links.pkl', 'wb')
+        cPickle.dump(self.data, output, -1)
+        
+        self.lock.release()
+    
+    def queueEmpty(self):
+        return (self.data["queue"] == [])
+    
+    def getDownloadList(self):
+        """
+            for thread_list only
+        """
+        files = []
+        for pypack in self.data["queue"]:
+            for pyfile in pypack.files:
+                if pyfile.status.type == "reconnected" or pyfile.status.type == None:
+                    files.append(pyfile)
+        return files
+    
+    class pyLoadCollector():
+        def __init__(collector, file_list):
+            collector.file_list = file_list
+        
+        def _getFileFromID(collector, id):
+            """
+                returns PyLoadFile instance and position in collector with given id
+            """
+            for n, pyfile in enumerate(collector.file_list.data["collector"]):
+                if pyfile.id == id:
+                    return (n, pyfile)
+            raise NoSuchElementException()
+        
+        def _getFreeID(collector):
+            """
+                returns a free id
+            """
+            ids = []
+            for pyfile in collector.file_list.data["collector"]:
+                ids.append(pyfile.id)
+            id = 1
+            while id in ids:
+                id += 1
+            return id
+        
+        def getFile(collector, id):
+            """
+                returns PyLoadFile instance from given id
+            """
+            return collector._getFileFromID(id)[1]
+            
+        def popFile(collector, id):
+            """
+                returns PyLoadFile instance given id and remove it from the collector
+            """
+            collector.file_list.lock.acquire()
+            try:
+                n, pyfile = collector._getFileFromID(id)
+                del collector.file_list.data["collector"][n]
+                collector.file_list.lock.release()
+            except:
+                collector.file_list.lock.release()
+            else:
+                return pyfile
+        
+        def addLink(collector, url):
+            """
+                appends a new PyLoadFile instance to the end of the collector
+            """
+            pyfile = PyLoadFile(url)
+            pyfile.id = collector._getFreeID()
+            pyfile.download_folder =  collector.file_list.download_folder
+            collector.file_list.lock.acquire()
+            collector.file_list.data["collector"].append(pyfile)
+            collector.file_list.lock.release()
+            return pyfile.id
+        
+        def removeFile(collector, id):
+            """
+                removes PyLoadFile instance with the given id from collector
+            """
+            collector.popFile(id)
+        
+        def replaceFile(collector, newpyfile):
+            """
+                replaces PyLoadFile instance with the given PyLoadFile instance at the given id
+            """
+            collector.file_list.lock.acquire()
+            try:
+                n, pyfile = collector._getFileFromID(newpyfile.id)
+                collector.file_list.data["collector"][n] = newpyfile
+            finally:
+                collector.file_list.lock.release()
+        
+    class pyLoadPackager():
+        def __init__(packager, file_list):
+            packager.file_list = file_list
+        
+        def _getFreeID(packager):
+            """
+                returns a free id
+            """
+            ids = []
+            for pypack in (packager.file_list.data["packages"] + packager.file_list.data["queue"]):
+                ids.append(pypack.id)
+            id = 1
+            while id in ids:
+                id += 1
+            return id
+        
+        def _getPackageFromID(packager, id):
+            """
+                returns PyLoadPackage instance and position with given id
+            """
+            for n, pypack in enumerate(packager.file_list.data["packages"]):
+                if pypack.id == id:
+                    return ("packages", n, pypack)
+            for n, pypack in enumerate(packager.file_list.data["queue"]):
+                if pypack.id == id:
+                    return ("queue", n, pypack)
+            raise NoSuchElementException()
+        
+        def addNewPackage(packager, package_name=None):
+            pypack = PyLoadPackage()
+            pypack.id = packager._getFreeID()
+            if package_name is not None:
+                pypack.data["package_name"] = package_name
+            packager.file_list.data["packages"].append(pypack)
+            return pypack.id
+        
+        def removePackage(packager, id):
+            packager.file_list.lock.acquire()
+            try:
+                key, n, pypack = packager._getPackageFromID(id)
+                del packager.file_list.data[key][n]
+            finally:
+                packager.file_list.lock.release()
+        
+        def pushPackage2Queue(packager, id):
+            packager.file_list.lock.acquire()
+            try:
+                key, n, pypack = packager._getPackageFromID(id)
+                if key == "packages":
+                    del packager.file_list.data["packages"][n]
+                    packager.file_list.data["queue"].append(pypack)
+            finally:
+                packager.file_list.lock.release()
+        
+        def pullOutPackage(packager, id):
+            packager.file_list.lock.acquire()
+            try:
+                key, n, pypack = packager._getPackageFromID(id)
+                if key == "queue":
+                    del packager.file_list.data["queue"][n]
+                    packager.file_list.data["packages"].append(pypack)
+            finally:
+                packager.file_list.lock.release()
+        
+        def setPackageData(packager, id, package_name=None, folder=None):
+            packager.file_list.lock.acquire()
+            try:
+                key, n, pypack = packager._getPackageFromID(id)
+                if package_name is not None:
+                    pypack.data["package_name"] = package_name
+                if folder is not None:
+                    pypack.data["folder"] = folder
+                packager.file_list.data[key][n] = pypack
+            finally:
+                packager.file_list.lock.release()
+        
+        def addFileToPackage(packager, id, pyfile):
+            key, n, pypack = packager._getPackageFromID(id)
+            pyfile.package = pypack
+            pypack.files.append(pyfile)
+            packager.file_list.data[key][n] = pypack
+       
+        def removeFileFromPackage(packager, id, pid):
+            key, n, pypack = packager._getPackageFromID(pid)
+            for k, pyfile in enumerate(pypack.files):
+                if id == pyfile.id:
+                    del pypack.files[k]
+                    return True
+            raise NoSuchElementException()
 
-class Data():
-    def __init__(self, url, folder=""):
-        self.url = url
-        self.folder = folder
+class PyLoadPackage():
+    def __init__(self):
+        self.files = []
+        self.data = {
+            "id": None,
+            "package_name": "",
+            "folder": ""
+        }
 
-class PyLoadFile:
-    """ represents the url or file
-    """
-    def __init__(self, parent, url):
-        self.parent = parent
+class PyLoadFile():
+    def __init__(self, url):
         self.id = None
         self.url = url
         self.folder = None
+        self.package = None
         self.filename = "filename"
         self.download_folder = ""
+        self.active = False
         pluginName = self._get_my_plugin()
         if pluginName:
             self.modul = __import__(pluginName)
@@ -163,18 +278,12 @@ class PyLoadFile:
             pluginClass = module.Plugin.Plugin
         self.plugin = pluginClass(self)
         self.status = Status(self)
+    
     def _get_my_plugin(self):
-        
-        """ searches the right plugin for an url
-        """
         for plugin, plugin_pattern in self.parent.plugins_avaible.items():
             if re.match(plugin_pattern, self.url) != None:
                 return plugin
 
     def init_download(self):
-
         if self.parent.config['proxy']['activated']:
             self.plugin.req.add_proxy(self.parent.config['proxy']['protocol'], self.parent.config['proxy']['adress'])
-
-        #@TODO: check dependicies, ocr etc
-
diff --git a/module/remote/ClientHandler.py b/module/remote/ClientHandler.py
deleted file mode 100644
index 9653db9ae..000000000
--- a/module/remote/ClientHandler.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-authored by:  RaNaN
-
-this module handels the incoming requests
-
-"""
-import hashlib
-
-from Crypto.Cipher import Blowfish
-from RequestHandler import RequestHandler
-
-class ClientHandler(RequestHandler):
-    def __init__(self, client, pw):
-        self.client = client
-        key = hashlib.sha256(pw)
-        self.bf = Blowfish.new(key.hexdigest(), Blowfish.MODE_ECB)
-
-    def proceed(self, data):
-        obj = self.decrypt(data)
-
-        self.client.data_arrived(obj)
\ No newline at end of file
diff --git a/module/remote/ClientSocket.py b/module/remote/ClientSocket.py
deleted file mode 100644
index 0e6a5ed53..000000000
--- a/module/remote/ClientSocket.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-authored by:  RaNaN
-
-socket for connecting to the core's server
-
-"""
-import asynchat
-import asyncore
-import socket
-import threading
-
-from ClientHandler import ClientHandler
-from RequestObject import RequestObject
-
-class SocketThread(threading.Thread):
-    def __init__(self, adress, port, pw, client):
-        threading.Thread.__init__(self)
-        self.setDaemon(True)
-        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        sock.connect((adress, port))
-        self.socket = ClientSocket(sock, pw, client)
-        self.start()
-
-    def run(self):
-        asyncore.loop()
-        print "loop closed"
-
-    def push_exec(self, function, args=[]):
-        obj = RequestObject()
-        obj.command = "exec"
-        obj.function = function
-        obj.args = args
-        self.push(obj)
-
-    def push(self, obj):
-        self.socket.push_obj(obj)
-
-
-class ClientSocket(asynchat.async_chat):
-    def __init__(self, sock, pw, client):
-        asynchat.async_chat.__init__(self, sock)
-        self.data = ""
-        self.handler = ClientHandler(client, pw)
-        self.set_terminator("\n")
-        #self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
-
-    def handle_close(self):
-        print "Disconnected from", self.getpeername()
-        self.close()
-
-    def collect_incoming_data(self, data):
-        self.data += data
-
-    def found_terminator(self):
-        self.handler.proceed(self.data)
-        self.data = ""
-
-    def push_obj(self, obj):
-        string = self.handler.encrypt(obj)
-        self.push(string)
\ No newline at end of file
diff --git a/module/remote/RequestHandler.py b/module/remote/RequestHandler.py
deleted file mode 100644
index a879d2297..000000000
--- a/module/remote/RequestHandler.py
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-authored by:  RaNaN
-
-this module handels the incoming requests
-
-"""
-
-import base64
-import hashlib
-import random
-import string
-
-import cPickle
-from Crypto.Cipher import Blowfish
-from RequestObject import RequestObject
-
-
-class RequestHandler:
-    def __init__(self, core):
-        self.core = core
-        key = hashlib.sha256(core.config['remote']['password'])
-        self.bf = Blowfish.new(key.hexdigest(), Blowfish.MODE_ECB)
-
-    def proceed(self, data):
-        obj = self.decrypt(data)
-
-        if obj.command == "exec":
-            func = getattr(self.core, obj.function)
-            obj.data = func( * obj.args)
-        else:
-            obj.data = "error happend"
-
-        return self.encrypt(obj)
-
-
-    def decrypt(self, dec_str):
-        try:
-            dec_str = base64.standard_b64decode(dec_str)
-            dec_str = self.bf.decrypt(dec_str)
-
-            dec_str = dec_str[:-(int(dec_str[-1], 16) + 1)]
-            obj = cPickle.loads(dec_str)
-        except:
-            obj = RequestObject()
-
-        return obj
-
-    def encrypt(self, obj):
-        enc_str = cPickle.dumps(obj, 1)
-        padding = len(enc_str) % 16
-        padding = 16 - padding
-
-        p_str = ""
-        for i in range(padding - 1):
-            p_str += random.choice(string.letters + string.digits + string.punctuation)
-        p_str += hex(len(p_str)).replace("0x", "")
-        enc_str += p_str
-
-        enc_str = self.bf.encrypt(enc_str)
-        enc_str = base64.standard_b64encode(enc_str)
-        return enc_str + "\n"
diff --git a/module/remote/RequestObject.py b/module/remote/RequestObject.py
deleted file mode 100644
index 54ea837a6..000000000
--- a/module/remote/RequestObject.py
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-authored by:  RaNaN
-
-represents the object for interaction
-
-"""
-class RequestObject(object):
-    def __init__(self):
-        self.version = 0
-        self.sender = "ip"
-        self.status = {}
-        self.command = None
-        self.function = ""
-        self.args = []
-        self.data = None
diff --git a/module/thread_list.py b/module/thread_list.py
index da13afba6..aa50a654f 100644
--- a/module/thread_list.py
+++ b/module/thread_list.py
@@ -60,7 +60,7 @@ class Thread_List(object):
     def get_job(self):
         """return job if suitable, otherwise send thread idle"""
 
-        if not self.parent.is_time_download() or self.pause or self.reconnecting or not self.list.files: #conditions when threads dont download
+        if not self.parent.is_time_download() or self.pause or self.reconnecting or self.list.queueEmpty(): #conditions when threads dont download
             return None
 
         self.init_reconnect()
@@ -68,15 +68,16 @@ class Thread_List(object):
         self.lock.acquire()
 
         pyfile = None
-        for i in range(len(self.list.files)):
-            if not self.list.files[i].modul.__name__ in self.occ_plugins:
-                pyfile = self.list.files.pop(i)
+        for f in self.list.getDownloadList():
+            if not f.modul.__name__ in self.occ_plugins:
+                pyfile = f
                 break
 
         if pyfile:
             self.py_downloading.append(pyfile)
             if not pyfile.plugin.multi_dl:
                 self.occ_plugins.append(pyfile.modul.__name__)
+            pyfile.active = True
             self.parent.logger.info('Download starts: ' + pyfile.url)
 
         self.lock.release()
@@ -89,6 +90,8 @@ class Thread_List(object):
 
         if not pyfile.plugin.multi_dl:
             self.occ_plugins.remove(pyfile.modul.__name__)
+            
+        pyfile.active = False
 
         if pyfile.plugin.req.curl and not pyfile.status == "reconnected":
             try:
@@ -101,25 +104,23 @@ class Thread_List(object):
         if pyfile.status.type == "finished":
             self.parent.logger.info('Download finished: ' + pyfile.url + ' @' + str(pyfile.status.get_speed()) + 'kb/s')
 
-            self.list.remove(pyfile)
-
             if pyfile.plugin.props['type'] == "container":
-                self.list.extend(pyfile.plugin.links)
-
+                self.list.packager.removeFileFromPackage(pyfile.id, pyfile.package.id)
+                for link in pyfile.plugin.links:
+                    id = self.list.collector.addLink(link)
+                    pyfile.packager.pullOutPackage(pyfile.package.id)
+                    pyfile.packager.addFileToPackage(pyfile.package.id, pyfile.collector.popFile(id))
 
-        elif pyfile.status.type == "reconnected":#put it back in queque
+        elif pyfile.status.type == "reconnected":
             pyfile.plugin.req.init_curl()
-            self.list.files.insert(0, pyfile)
 
         elif pyfile.status.type == "failed":
             self.parent.logger.warning("Download failed: " + pyfile.url+ " | " + pyfile.status.error)
             with open(self.parent.config['general']['failed_file'], 'a') as f:
                 f.write(pyfile.url + "\n")
-            self.list.remove(pyfile)
 
         elif pyfile.status.type == "aborted":
             self.parent.logger.info("Download aborted: " + pyfile.url)
-            self.list.remove(pyfile)
 
         self.list.save()