Move base plugins to internal folder

6 files changed, 1250 insertions, 0 deletions

diff --git a/module/plugins/internal/Account.py b/module/plugins/internal/Account.py
new file mode 100644
index 000000000..c147404e0
--- /dev/null
+++ b/module/plugins/internal/Account.py
@@ -0,0 +1,292 @@
+# -*- coding: utf-8 -*-
+
+"""
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License,
+    or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, see <http://www.gnu.org/licenses/>.
+    
+    @author: mkaay
+"""
+
+from random import choice
+from time import time
+from traceback import print_exc
+from threading import RLock
+
+from Plugin import Base
+from module.utils import compare_time, parseFileSize, lock
+
+class WrongPassword(Exception):
+    pass
+
+
+class Account(Base):
+    """
+    Base class for every Account plugin.
+    Just overwrite `login` and cookies will be stored and account becomes accessible in\
+    associated hoster plugin. Plugin should also provide `loadAccountInfo`
+    """
+    __name__ = "Account"
+    __version__ = "0.2"
+    __type__ = "account"
+    __description__ = """Account Plugin"""
+    __author_name__ = ("mkaay")
+    __author_mail__ = ("mkaay@mkaay.de")
+
+    #: after that time [in minutes] pyload will relogin the account
+    login_timeout = 600
+    #: account data will be reloaded after this time
+    info_threshold = 600
+
+
+    def __init__(self, manager, accounts):
+        Base.__init__(self, manager.core)
+
+        self.manager = manager
+        self.accounts = {}
+        self.infos = {} # cache for account information
+        self.lock = RLock()
+
+        self.timestamps = {}
+        self.setAccounts(accounts)
+        self.init()
+
+    def init(self):
+        pass
+
+    def login(self, user, data, req):
+        """login into account, the cookies will be saved so user can be recognized
+
+        :param user: loginname
+        :param data: data dictionary
+        :param req: `Request` instance
+        """
+        pass
+
+    @lock
+    def _login(self, user, data):
+        # set timestamp for login
+        self.timestamps[user] = time()
+        
+        req = self.getAccountRequest(user)
+        try:
+            self.login(user, data, req)
+        except WrongPassword:
+            self.logWarning(
+                _("Could not login with account %(user)s | %(msg)s") % {"user": user
+                                                                        , "msg": _("Wrong Password")})
+            data["valid"] = False
+
+        except Exception, e:
+            self.logWarning(
+                _("Could not login with account %(user)s | %(msg)s") % {"user": user
+                                                                        , "msg": e})
+            data["valid"] = False
+            if self.core.debug:
+                print_exc()
+        finally:
+            if req: req.close()
+
+    def relogin(self, user):
+        req = self.getAccountRequest(user)
+        if req:
+            req.cj.clear()
+            req.close()
+        if user in self.infos:
+            del self.infos[user] #delete old information
+
+        self._login(user, self.accounts[user])
+
+    def setAccounts(self, accounts):
+        self.accounts = accounts
+        for user, data in self.accounts.iteritems():
+            self._login(user, data)
+            self.infos[user] = {}
+
+    def updateAccounts(self, user, password=None, options={}):
+        """ updates account and return true if anything changed """
+
+        if user in self.accounts:
+            self.accounts[user]["valid"] = True #do not remove or accounts will not login
+            if password:
+                self.accounts[user]["password"] = password
+                self.relogin(user)
+                return True
+            if options:
+                before = self.accounts[user]["options"]
+                self.accounts[user]["options"].update(options)
+                return self.accounts[user]["options"] != before
+        else:
+            self.accounts[user] = {"password": password, "options": options, "valid": True}
+            self._login(user, self.accounts[user])
+            return True
+
+    def removeAccount(self, user):
+        if user in self.accounts:
+            del self.accounts[user]
+        if user in self.infos:
+            del self.infos[user]
+        if user in self.timestamps:
+            del self.timestamps[user]
+
+    @lock
+    def getAccountInfo(self, name, force=False):
+        """retrieve account infos for an user, do **not** overwrite this method!\\
+        just use it to retrieve infos in hoster plugins. see `loadAccountInfo`
+
+        :param name: username
+        :param force: reloads cached account information
+        :return: dictionary with information
+        """
+        data = Account.loadAccountInfo(self, name)
+
+        if force or name not in self.infos:
+            self.logDebug("Get Account Info for %s" % name)
+            req = self.getAccountRequest(name)
+
+            try:
+                infos = self.loadAccountInfo(name, req)
+                if not type(infos) == dict:
+                    raise Exception("Wrong return format")
+            except Exception, e:
+                infos = {"error": str(e)}
+
+            if req: req.close()
+
+            self.logDebug("Account Info: %s" % str(infos))
+
+            infos["timestamp"] = time()
+            self.infos[name] = infos
+        elif "timestamp" in self.infos[name] and self.infos[name][
+                                                       "timestamp"] + self.info_threshold * 60 < time():
+            self.logDebug("Reached timeout for account data")
+            self.scheduleRefresh(name)
+
+        data.update(self.infos[name])
+        return data
+
+    def isPremium(self, user):
+        info = self.getAccountInfo(user)
+        return info["premium"]
+
+    def loadAccountInfo(self, name, req=None):
+        """this should be overwritten in account plugin,\
+        and retrieving account information for user
+
+        :param name:
+        :param req: `Request` instance
+        :return:
+        """
+        return {
+            "validuntil": None, # -1 for unlimited
+            "login": name,
+            #"password": self.accounts[name]["password"], #@XXX: security
+            "options": self.accounts[name]["options"],
+            "valid": self.accounts[name]["valid"],
+            "trafficleft": None, # in kb, -1 for unlimited
+            "maxtraffic": None,
+            "premium": True, #useful for free accounts
+            "timestamp": 0, #time this info was retrieved
+            "type": self.__name__,
+            }
+
+    def getAllAccounts(self, force=False):
+        return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()]
+
+    def getAccountRequest(self, user=None):
+        if not user:
+            user, data = self.selectAccount()
+        if not user:
+            return None
+
+        req = self.core.requestFactory.getRequest(self.__name__, user)
+        return req
+
+    def getAccountCookies(self, user=None):
+        if not user:
+            user, data = self.selectAccount()
+        if not user:
+            return None
+
+        cj = self.core.requestFactory.getCookieJar(self.__name__, user)
+        return cj
+
+    def getAccountData(self, user):
+        return self.accounts[user]
+
+    def selectAccount(self):
+        """ returns an valid account name and data"""
+        usable = []
+        for user, data in self.accounts.iteritems():
+            if not data["valid"]: continue
+
+            if "time" in data["options"] and data["options"]["time"]:
+                time_data = ""
+                try:
+                    time_data = data["options"]["time"][0]
+                    start, end = time_data.split("-")
+                    if not compare_time(start.split(":"), end.split(":")):
+                        continue
+                except:
+                    self.logWarning(_("Your Time %s has wrong format, use: 1:22-3:44") % time_data)
+
+            if user in self.infos:
+                if "validuntil" in self.infos[user]:
+                    if self.infos[user]["validuntil"] > 0 and time() > self.infos[user]["validuntil"]:
+                        continue
+                if "trafficleft" in self.infos[user]:
+                    if self.infos[user]["trafficleft"] == 0:
+                        continue
+
+            usable.append((user, data))
+
+        if not usable: return None, None
+        return choice(usable)
+
+    def canUse(self):
+        return False if self.selectAccount() == (None, None) else True
+
+    def parseTraffic(self, string): #returns kbyte
+        return parseFileSize(string) / 1024
+
+    def wrongPassword(self):
+        raise WrongPassword
+
+    def empty(self, user):
+        if user in self.infos:
+            self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user)
+
+            self.infos[user].update({"trafficleft": 0})
+            self.scheduleRefresh(user, 30 * 60)
+
+    def expired(self, user):
+        if user in self.infos:
+            self.logWarning(_("Account %s is expired, checking again in 1h") % user)
+
+            self.infos[user].update({"validuntil": time() - 1})
+            self.scheduleRefresh(user, 60 * 60)
+
+    def scheduleRefresh(self, user, time=0, force=True):
+        """ add task to refresh account info to sheduler """
+        self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time))
+        self.core.scheduler.addJob(time, self.getAccountInfo, [user, force])
+
+    @lock
+    def checkLogin(self, user):
+        """ checks if user is still logged in """
+        if user in self.timestamps:
+            if self.timestamps[user] + self.login_timeout * 60 < time():
+                self.logDebug("Reached login timeout for %s" % user)
+                self.relogin(user)
+                return False
+
+        return True
diff --git a/module/plugins/internal/Container.py b/module/plugins/internal/Container.py
new file mode 100644
index 000000000..c233d3710
--- /dev/null
+++ b/module/plugins/internal/Container.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+
+"""
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License,
+    or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, see <http://www.gnu.org/licenses/>.
+    
+    @author: mkaay
+"""
+
+from module.plugins.Crypter import Crypter
+
+from os.path import join, exists, basename
+from os import remove
+import re
+
+class Container(Crypter):
+    __name__ = "Container"
+    __version__ = "0.1"
+    __pattern__ = None
+    __type__ = "container"
+    __description__ = """Base container plugin"""
+    __author_name__ = ("mkaay")
+    __author_mail__ = ("mkaay@mkaay.de")
+
+
+    def preprocessing(self, thread):
+        """prepare"""
+
+        self.setup()
+        self.thread = thread
+        
+        self.loadToDisk()
+
+        self.decrypt(self.pyfile)
+        self.deleteTmp()
+        
+        self.createPackages()
+    
+
+    def loadToDisk(self):
+        """loads container to disk if its stored remotely and overwrite url, 
+        or check existent on several places at disk"""
+        
+        if self.pyfile.url.startswith("http"):
+            self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1]
+            content = self.load(self.pyfile.url)
+            self.pyfile.url = join(self.config["general"]["download_folder"], self.pyfile.name)
+            f = open(self.pyfile.url, "wb" )
+            f.write(content)
+            f.close()
+            
+        else:
+            self.pyfile.name = basename(self.pyfile.url)
+            if not exists(self.pyfile.url):
+                if exists(join(pypath, self.pyfile.url)):
+                    self.pyfile.url = join(pypath, self.pyfile.url)
+                else:
+                    self.fail(_("File not exists."))
+      
+
+    def deleteTmp(self):
+        if self.pyfile.name.startswith("tmp_"):
+            remove(self.pyfile.url)
+
+        
diff --git a/module/plugins/internal/Crypter.py b/module/plugins/internal/Crypter.py
new file mode 100644
index 000000000..d1549fe80
--- /dev/null
+++ b/module/plugins/internal/Crypter.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+
+"""
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License,
+    or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, see <http://www.gnu.org/licenses/>.
+    
+    @author: mkaay
+"""
+
+from module.plugins.Plugin import Plugin
+
+class Crypter(Plugin):
+    __name__ = "Crypter"
+    __version__ = "0.1"
+    __pattern__ = None
+    __type__ = "container"
+    __description__ = """Base crypter plugin"""
+    __author_name__ = ("mkaay")
+    __author_mail__ = ("mkaay@mkaay.de")
+    
+    def __init__(self, pyfile):
+        Plugin.__init__(self, pyfile)
+        
+        #: Put all packages here. It's a list of tuples like: ( name, [list of links], folder )
+        self.packages = []
+
+        #: List of urls, pyLoad will generate packagenames
+        self.urls = []
+        
+        self.multiDL = True
+        self.limitDL = 0
+    
+
+    def preprocessing(self, thread):
+        """prepare"""
+        self.setup()
+        self.thread = thread
+
+        self.decrypt(self.pyfile)
+        
+        self.createPackages()
+        
+
+    def decrypt(self, pyfile):
+        raise NotImplementedError
+
+    def createPackages(self):
+        """ create new packages from self.packages """
+        for pack in self.packages:
+
+            self.log.debug("Parsed package %(name)s with %(len)d links" % { "name" : pack[0], "len" : len(pack[1]) } )
+            
+            links = [x.decode("utf-8") for x in pack[1]]
+            
+            pid = self.core.api.addPackage(pack[0], links, self.pyfile.package().queue)
+
+            if self.pyfile.package().password:
+                self.core.api.setPackageData(pid, {"password": self.pyfile.package().password})
+
+        if self.urls:
+            self.core.api.generateAndAddPackages(self.urls)
+            
diff --git a/module/plugins/internal/Hook.py b/module/plugins/internal/Hook.py
new file mode 100644
index 000000000..5efd08bae
--- /dev/null
+++ b/module/plugins/internal/Hook.py
@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+
+"""
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License,
+    or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, see <http://www.gnu.org/licenses/>.
+    
+    @author: mkaay
+    @interface-version: 0.2
+"""
+
+from traceback import print_exc
+
+from Plugin import Base
+
+class Expose(object):
+    """ used for decoration to declare rpc services """
+
+    def __new__(cls, f, *args, **kwargs):
+        hookManager.addRPC(f.__module__, f.func_name, f.func_doc)
+        return f
+
+def threaded(f):
+    def run(*args,**kwargs):
+        hookManager.startThread(f, *args, **kwargs)
+    return run
+
+class Hook(Base):
+    """
+    Base class for hook plugins.
+    """
+    __name__ = "Hook"
+    __version__ = "0.2"
+    __type__ = "hook"
+    __threaded__ = []
+    __config__ = [ ("name", "type", "desc" , "default") ]
+    __description__ = """interface for hook"""
+    __author_name__ = ("mkaay", "RaNaN")
+    __author_mail__ = ("mkaay@mkaay.de", "RaNaN@pyload.org")
+
+    #: automatically register event listeners for functions, attribute will be deleted dont use it yourself
+    event_map = None
+
+    # Alternative to event_map
+    #: List of events the plugin can handle, name the functions exactly like eventname.
+    event_list = None  # dont make duplicate entries in event_map
+
+
+    #: periodic call interval in secondc
+    interval = 60
+
+    def __init__(self, core, manager):
+        Base.__init__(self, core)
+
+        #: Provide information in dict here, usable by API `getInfo`
+        self.info = None
+
+        #: Callback of periodical job task, used by hookmanager
+        self.cb = None
+
+        #: `HookManager`
+        self.manager = manager
+
+        #register events
+        if self.event_map:
+            for event, funcs in self.event_map.iteritems():
+                if type(funcs) in (list, tuple):
+                    for f in funcs:
+                        self.manager.addEvent(event, getattr(self,f))
+                else:
+                    self.manager.addEvent(event, getattr(self,funcs))
+
+            #delete for various reasons
+            self.event_map = None
+
+        if self.event_list:
+            for f in self.event_list:
+                self.manager.addEvent(f, getattr(self,f))
+
+            self.event_list = None
+
+        self.initPeriodical()
+        self.setup()
+
+    def initPeriodical(self):
+        if self.interval >=1:
+            self.cb = self.core.scheduler.addJob(0, self._periodical, threaded=False)
+
+    def _periodical(self):
+        try:
+            if self.isActivated(): self.periodical()
+        except Exception, e:
+            self.core.log.error(_("Error executing hooks: %s") % str(e))
+            if self.core.debug:
+                print_exc()
+
+        self.cb = self.core.scheduler.addJob(self.interval, self._periodical, threaded=False)
+
+
+    def __repr__(self):
+        return "<Hook %s>" % self.__name__
+               
+    def setup(self):
+        """ more init stuff if needed """
+        pass
+
+    def unload(self):
+        """ called when hook was deactivated """
+        pass
+    
+    def isActivated(self):
+        """ checks if hook is activated"""
+        return self.config.getPlugin(self.__name__, "activated")
+    
+
+    #event methods - overwrite these if needed    
+    def coreReady(self):
+        pass
+
+    def coreExiting(self):
+        pass
+    
+    def downloadPreparing(self, pyfile):
+        pass
+    
+    def downloadFinished(self, pyfile):
+        pass
+    
+    def downloadFailed(self, pyfile):
+        pass
+    
+    def packageFinished(self, pypack):
+        pass
+
+    def beforeReconnecting(self, ip):
+        pass
+    
+    def afterReconnecting(self, ip):
+        pass
+    
+    def periodical(self):
+        pass
+
+    def newCaptchaTask(self, task):
+        """ new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """
+        pass
+
+    def captchaCorrect(self, task):
+        pass
+
+    def captchaInvalid(self, task):
+        pass
\ No newline at end of file
diff --git a/module/plugins/internal/Hoster.py b/module/plugins/internal/Hoster.py
new file mode 100644
index 000000000..814a70949
--- /dev/null
+++ b/module/plugins/internal/Hoster.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+
+"""
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License,
+    or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, see <http://www.gnu.org/licenses/>.
+    
+    @author: mkaay
+"""
+
+from module.plugins.Plugin import Plugin
+
+def getInfo(self):
+        #result = [ .. (name, size, status, url) .. ]
+        return
+
+class Hoster(Plugin):
+    __name__ = "Hoster"
+    __version__ = "0.1"
+    __pattern__ = None
+    __type__ = "hoster"
+    __description__ = """Base hoster plugin"""
+    __author_name__ = ("mkaay")
+    __author_mail__ = ("mkaay@mkaay.de")
diff --git a/module/plugins/internal/Plugin.py b/module/plugins/internal/Plugin.py
new file mode 100644
index 000000000..15bf3971f
--- /dev/null
+++ b/module/plugins/internal/Plugin.py
@@ -0,0 +1,617 @@
+# -*- coding: utf-8 -*-
+
+"""
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 3 of the License,
+    or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+    See the GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+    @author: RaNaN, spoob, mkaay
+"""
+
+from time import time, sleep
+from random import randint
+
+import os
+from os import remove, makedirs, chmod, stat
+from os.path import exists, join
+
+if os.name != "nt":
+    from os import chown
+    from pwd import getpwnam
+    from grp import getgrnam
+
+from itertools import islice
+
+from module.utils import save_join, save_path, fs_encode, fs_decode
+
+def chunks(iterable, size):
+    it = iter(iterable)
+    item = list(islice(it, size))
+    while item:
+        yield item
+        item = list(islice(it, size))
+
+
+class Abort(Exception):
+    """ raised when aborted """
+
+
+class Fail(Exception):
+    """ raised when failed """
+
+
+class Reconnect(Exception):
+    """ raised when reconnected """
+
+
+class Retry(Exception):
+    """ raised when start again from beginning """
+
+
+class SkipDownload(Exception):
+    """ raised when download should be skipped """
+
+
+class Base(object):
+    """
+    A Base class with log/config/db methods *all* plugin types can use
+    """
+
+    def __init__(self, core):
+        #: Core instance
+        self.core = core
+        #: logging instance
+        self.log = core.log
+        #: core config
+        self.config = core.config
+
+    #log functions
+    def logInfo(self, *args):
+        self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
+
+    def logWarning(self, *args):
+        self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
+
+    def logError(self, *args):
+        self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
+
+    def logDebug(self, *args):
+        self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
+
+
+    def setConf(self, option, value):
+        """ see `setConfig` """
+        self.core.config.setPlugin(self.__name__, option, value)
+
+    def setConfig(self, option, value):
+        """ Set config value for current plugin
+
+        :param option:
+        :param value:
+        :return:
+        """
+        self.setConf(option, value)
+
+    def getConf(self, option):
+        """ see `getConfig` """
+        return self.core.config.getPlugin(self.__name__, option)
+
+    def getConfig(self, option):
+        """ Returns config value for current plugin
+
+        :param option:
+        :return:
+        """
+        return self.getConf(option)
+
+    def setStorage(self, key, value):
+        """ Saves a value persistently to the database """
+        self.core.db.setStorage(self.__name__, key, value)
+
+    def store(self, key, value):
+        """ same as `setStorage` """
+        self.core.db.setStorage(self.__name__, key, value)
+
+    def getStorage(self, key=None, default=None):
+        """ Retrieves saved value or dict of all saved entries if key is None """
+        if key is not None:
+            return self.core.db.getStorage(self.__name__, key) or default
+        return self.core.db.getStorage(self.__name__, key)
+
+    def retrieve(self, *args, **kwargs):
+        """ same as `getStorage` """
+        return self.getStorage(*args, **kwargs)
+
+    def delStorage(self, key):
+        """ Delete entry in db """
+        self.core.db.delStorage(self.__name__, key)
+
+
+class Plugin(Base):
+    """
+    Base plugin for hoster/crypter.
+    Overwrite `process` / `decrypt` in your subclassed plugin.
+    """
+    __name__ = "Plugin"
+    __version__ = "0.4"
+    __pattern__ = None
+    __type__ = "hoster"
+    __config__ = [("name", "type", "desc", "default")]
+    __description__ = """Base Plugin"""
+    __author_name__ = ("RaNaN", "spoob", "mkaay")
+    __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de")
+
+    def __init__(self, pyfile):
+        Base.__init__(self, pyfile.m.core)
+
+        self.wantReconnect = False
+        #: enables simultaneous processing of multiple downloads
+        self.multiDL = True
+        self.limitDL = 0
+        #: chunk limit
+        self.chunkLimit = 1
+        self.resumeDownload = False
+
+        #: time() + wait in seconds
+        self.waitUntil = 0
+        self.waiting = False
+
+        self.ocr = None  #captcha reader instance
+        #: account handler instance, see :py:class:`Account`
+        self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__)
+
+        #: premium status
+        self.premium = False
+        #: username/login
+        self.user = None
+
+        if self.account and not self.account.canUse(): self.account = None
+        if self.account:
+            self.user, data = self.account.selectAccount()
+            #: Browser instance, see `network.Browser`
+            self.req = self.account.getAccountRequest(self.user)
+            self.chunkLimit = -1 # chunk limit, -1 for unlimited
+            #: enables resume (will be ignored if server dont accept chunks)
+            self.resumeDownload = True
+            self.multiDL = True  #every hoster with account should provide multiple downloads
+            #: premium status
+            self.premium = self.account.isPremium(self.user)
+        else:
+            self.req = pyfile.m.core.requestFactory.getRequest(self.__name__)
+
+        #: associated pyfile instance, see `PyFile`
+        self.pyfile = pyfile
+        self.thread = None # holds thread in future
+
+        #: location where the last call to download was saved
+        self.lastDownload = ""
+        #: re match of the last call to `checkDownload`
+        self.lastCheck = None
+        #: js engine, see `JsEngine`
+        self.js = self.core.js
+        self.cTask = None #captcha task
+
+        self.retries = 0 # amount of retries already made
+        self.html = None # some plugins store html code here
+
+        self.init()
+
+    def getChunkCount(self):
+        if self.chunkLimit <= 0:
+            return self.config["download"]["chunks"]
+        return min(self.config["download"]["chunks"], self.chunkLimit)
+
+    def __call__(self):
+        return self.__name__
+
+    def init(self):
+        """initialize the plugin (in addition to `__init__`)"""
+        pass
+
+    def setup(self):
+        """ setup for enviroment and other things, called before downloading (possibly more than one time)"""
+        pass
+
+    def preprocessing(self, thread):
+        """ handles important things to do before starting """
+        self.thread = thread
+
+        if self.account:
+            self.account.checkLogin(self.user)
+        else:
+            self.req.clearCookies()
+
+        self.setup()
+
+        self.pyfile.setStatus("starting")
+
+        return self.process(self.pyfile)
+
+
+    def process(self, pyfile):
+        """the 'main' method of every plugin, you **have to** overwrite it"""
+        raise NotImplementedError
+
+    def resetAccount(self):
+        """ dont use account and retry download """
+        self.account = None
+        self.req = self.core.requestFactory.getRequest(self.__name__)
+        self.retry()
+
+    def checksum(self, local_file=None):
+        """
+        return codes:
+        0  - checksum ok
+        1  - checksum wrong
+        5  - can't get checksum
+        10 - not implemented
+        20 - unknown error
+        """
+        #@TODO checksum check hook
+
+        return True, 10
+
+
+    def setWait(self, seconds, reconnect=False):
+        """Set a specific wait time later used with `wait`
+        
+        :param seconds: wait time in seconds
+        :param reconnect: True if a reconnect would avoid wait time
+        """
+        if reconnect:
+            self.wantReconnect = True
+        self.pyfile.waitUntil = time() + int(seconds)
+
+    def wait(self):
+        """ waits the time previously set """
+        self.waiting = True
+        self.pyfile.setStatus("waiting")
+
+        while self.pyfile.waitUntil > time():
+            self.thread.m.reconnecting.wait(2)
+
+            if self.pyfile.abort: raise Abort
+            if self.thread.m.reconnecting.isSet():
+                self.waiting = False
+                self.wantReconnect = False
+                raise Reconnect
+
+        self.waiting = False
+        self.pyfile.setStatus("starting")
+
+    def fail(self, reason):
+        """ fail and give reason """
+        raise Fail(reason)
+
+    def offline(self):
+        """ fail and indicate file is offline """
+        raise Fail("offline")
+
+    def tempOffline(self):
+        """ fail and indicates file ist temporary offline, the core may take consequences """
+        raise Fail("temp. offline")
+
+    def retry(self, max_tries=3, wait_time=1, reason=""):
+        """Retries and begin again from the beginning
+
+        :param max_tries: number of maximum retries
+        :param wait_time: time to wait in seconds
+        :param reason: reason for retrying, will be passed to fail if max_tries reached
+        """
+        if 0 < max_tries <= self.retries:
+            if not reason: reason = "Max retries reached"
+            raise Fail(reason)
+
+        self.wantReconnect = False
+        self.setWait(wait_time)
+        self.wait()
+
+        self.retries += 1
+        raise Retry(reason)
+
+    def invalidCaptcha(self):
+        if self.cTask:
+            self.cTask.invalid()
+
+    def correctCaptcha(self):
+        if self.cTask:
+            self.cTask.correct()
+
+    def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg',
+                       result_type='textual'):
+        """ Loads a captcha and decrypts it with ocr, plugin, user input
+
+        :param url: url of captcha image
+        :param get: get part for request
+        :param post: post part for request
+        :param cookies: True if cookies should be enabled
+        :param forceUser: if True, ocr is not used
+        :param imgtype: Type of the Image
+        :param result_type: 'textual' if text is written on the captcha\
+        or 'positional' for captcha where the user have to click\
+        on a specific region on the captcha
+        
+        :return: result of decrypting
+        """
+
+        img = self.load(url, get=get, post=post, cookies=cookies)
+
+        id = ("%.2f" % time())[-6:].replace(".", "")
+        temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb")
+        temp_file.write(img)
+        temp_file.close()
+
+        has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins
+
+        if self.core.captcha:
+            Ocr = self.core.pluginManager.loadClass("captcha", self.__name__)
+        else:
+            Ocr = None
+
+        if Ocr and not forceUser:
+            sleep(randint(3000, 5000) / 1000.0)
+            if self.pyfile.abort: raise Abort
+
+            ocr = Ocr()
+            result = ocr.get_captcha(temp_file.name)
+        else:
+            captchaManager = self.core.captchaManager
+            task = captchaManager.newTask(img, imgtype, temp_file.name, result_type)
+            self.cTask = task
+            captchaManager.handleCaptcha(task)
+
+            while task.isWaiting():
+                if self.pyfile.abort:
+                    captchaManager.removeTask(task)
+                    raise Abort
+                sleep(1)
+
+            captchaManager.removeTask(task)
+
+            if task.error and has_plugin: #ignore default error message since the user could use OCR
+                self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting"))
+            elif task.error:
+                self.fail(task.error)
+            elif not task.result:
+                self.fail(_("No captcha result obtained in appropiate time by any of the plugins."))
+
+            result = task.result
+            self.log.debug("Received captcha result: %s" % str(result))
+
+        if not self.core.debug:
+            try:
+                remove(temp_file.name)
+            except:
+                pass
+
+        return result
+
+
+    def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False):
+        """Load content at url and returns it
+
+        :param url:
+        :param get:
+        :param post:
+        :param ref:
+        :param cookies:
+        :param just_header: if True only the header will be retrieved and returned as dict
+        :param decode: Wether to decode the output according to http header, should be True in most cases
+        :return: Loaded content
+        """
+        if self.pyfile.abort: raise Abort
+        #utf8 vs decode -> please use decode attribute in all future plugins
+        if type(url) == unicode: url = str(url)
+
+        res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode)
+
+        if self.core.debug:
+            from inspect import currentframe
+
+            frame = currentframe()
+            if not exists(join("tmp", self.__name__)):
+                makedirs(join("tmp", self.__name__))
+
+            f = open(
+                join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
+                , "wb")
+            del frame # delete the frame or it wont be cleaned
+
+            try:
+                tmp = res.encode("utf8")
+            except:
+                tmp = res
+
+            f.write(tmp)
+            f.close()
+
+        if just_header:
+            #parse header
+            header = {"code": self.req.code}
+            for line in res.splitlines():
+                line = line.strip()
+                if not line or ":" not in line: continue
+
+                key, none, value = line.partition(":")
+                key = key.lower().strip()
+                value = value.strip()
+
+                if key in header:
+                    if type(header[key]) == list:
+                        header[key].append(value)
+                    else:
+                        header[key] = [header[key], value]
+                else:
+                    header[key] = value
+            res = header
+
+        return res
+
+    def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
+        """Downloads the content at url to download folder
+
+        :param url:
+        :param get:
+        :param post:
+        :param ref:
+        :param cookies:
+        :param disposition: if True and server provides content-disposition header\
+        the filename will be changed if needed
+        :return: The location where the file was saved
+        """
+
+        self.checkForSameFiles()
+
+        self.pyfile.setStatus("downloading")
+
+        download_folder = self.config['general']['download_folder']
+
+        location = save_join(download_folder, self.pyfile.package().folder)
+
+        if not exists(location):
+            makedirs(location, int(self.core.config["permission"]["folder"], 8))
+
+            if self.core.config["permission"]["change_dl"] and os.name != "nt":
+                try:
+                    uid = getpwnam(self.config["permission"]["user"])[2]
+                    gid = getgrnam(self.config["permission"]["group"])[2]
+
+                    chown(location, uid, gid)
+                except Exception, e:
+                    self.log.warning(_("Setting User and Group failed: %s") % str(e))
+
+        # convert back to unicode
+        location = fs_decode(location)
+        name = save_path(self.pyfile.name)
+
+        filename = join(location, name)
+
+        self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename)
+
+        try:
+            newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies,
+                                            chunks=self.getChunkCount(), resume=self.resumeDownload,
+                                            progressNotify=self.pyfile.setProgress, disposition=disposition)
+        finally:
+            self.pyfile.size = self.req.size
+
+        if disposition and newname and newname != name: #triple check, just to be sure
+            self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname})
+            self.pyfile.name = newname
+            filename = join(location, newname)
+
+        fs_filename = fs_encode(filename)
+
+        if self.core.config["permission"]["change_file"]:
+            chmod(fs_filename, int(self.core.config["permission"]["file"], 8))
+
+        if self.core.config["permission"]["change_dl"] and os.name != "nt":
+            try:
+                uid = getpwnam(self.config["permission"]["user"])[2]
+                gid = getgrnam(self.config["permission"]["group"])[2]
+
+                chown(fs_filename, uid, gid)
+            except Exception, e:
+                self.log.warning(_("Setting User and Group failed: %s") % str(e))
+
+        self.lastDownload = filename
+        return self.lastDownload
+
+    def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0):
+        """ checks the content of the last downloaded file, re match is saved to `lastCheck`
+        
+        :param rules: dict with names and rules to match (compiled regexp or strings)
+        :param api_size: expected file size
+        :param max_size: if the file is larger then it wont be checked
+        :param delete: delete if matched
+        :param read_size: amount of bytes to read from files larger then max_size
+        :return: dictionary key of the first rule that matched
+        """
+        lastDownload = fs_encode(self.lastDownload)
+        if not exists(lastDownload): return None
+
+        size = stat(lastDownload)
+        size = size.st_size
+
+        if api_size and api_size <= size: return None
+        elif size > max_size and not read_size: return None
+        self.log.debug("Download Check triggered")
+        f = open(lastDownload, "rb")
+        content = f.read(read_size if read_size else -1)
+        f.close()
+        #produces encoding errors, better log to other file in the future?
+        #self.log.debug("Content: %s" % content)
+        for name, rule in rules.iteritems():
+            if type(rule) in (str, unicode):
+                if rule in content:
+                    if delete:
+                        remove(lastDownload)
+                    return name
+            elif hasattr(rule, "search"):
+                m = rule.search(content)
+                if m:
+                    if delete:
+                        remove(lastDownload)
+                    self.lastCheck = m
+                    return name
+
+
+    def getPassword(self):
+        """ get the password the user provided in the package"""
+        password = self.pyfile.package().password
+        if not password: return ""
+        return password
+
+
+    def checkForSameFiles(self, starting=False):
+        """ checks if same file was/is downloaded within same package
+
+        :param starting: indicates that the current download is going to start
+        :raises SkipDownload:
+        """
+
+        pack = self.pyfile.package()
+
+        for pyfile in self.core.files.cache.values():
+            if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
+                if pyfile.status in (0, 12): #finished or downloading
+                    raise SkipDownload(pyfile.pluginname)
+                elif pyfile.status in (
+                5, 7) and starting: #a download is waiting/starting and was appenrently started before
+                    raise SkipDownload(pyfile.pluginname)
+
+        download_folder = self.config['general']['download_folder']
+        location = save_join(download_folder, pack.folder, self.pyfile.name)
+
+        if starting and self.core.config['download']['skip_existing'] and exists(location):
+            size = os.stat(location).st_size
+            if size >= self.pyfile.size:
+                raise SkipDownload("File exists.")
+
+        pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
+        if pyfile:
+            if exists(location):
+                raise SkipDownload(pyfile[0])
+
+            self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
+
+    def clean(self):
+        """ clean everything and remove references """
+        if hasattr(self, "pyfile"):
+            del self.pyfile
+        if hasattr(self, "req"):
+            self.req.close()
+            del self.req
+        if hasattr(self, "thread"):
+            del self.thread
+        if hasattr(self, "html"):
+            del self.html