From 052d09687627999b91b45a8c0224aa9de4769924 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Wed, 22 Dec 2010 14:25:56 +0100 Subject: cleaned request factory --- module/AccountManager.py | 179 --------------------- module/PluginManager.py | 329 --------------------------------------- module/RequestFactory.py | 94 ----------- module/network/Browser.py | 6 +- module/network/CookieJar.py | 41 +++++ module/network/RequestFactory.py | 68 ++++++++ module/plugins/AccountManager.py | 179 +++++++++++++++++++++ module/plugins/PluginManager.py | 329 +++++++++++++++++++++++++++++++++++++++ pyLoadCore.py | 9 +- 9 files changed, 626 insertions(+), 608 deletions(-) delete mode 100644 module/AccountManager.py delete mode 100644 module/PluginManager.py delete mode 100644 module/RequestFactory.py create mode 100644 module/network/CookieJar.py create mode 100644 module/network/RequestFactory.py create mode 100644 module/plugins/AccountManager.py create mode 100644 module/plugins/PluginManager.py diff --git a/module/AccountManager.py b/module/AccountManager.py deleted file mode 100644 index ab35334fc..000000000 --- a/module/AccountManager.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: RaNaN -""" - -from os.path import exists -from shutil import copy - -from module.PullEvents import AccountUpdateEvent - -ACC_VERSION = 1 - -######################################################################## -class AccountManager(): - """manages all accounts""" - - #---------------------------------------------------------------------- - def __init__(self, core): - """Constructor""" - - self.core = core - - self.accounts = {} # key = ( plugin ) - self.plugins = {} - - self.initAccountPlugins() - - self.loadAccounts() - - self.saveAccounts() # save to add categories to conf - - #---------------------------------------------------------------------- - def getAccountPlugin(self, plugin): - """get account instance for plugin or None if anonymous""" - if self.accounts.has_key(plugin): - if not self.plugins.has_key(plugin): - self.plugins[plugin] = self.core.pluginManager.getAccountPlugin(plugin)(self, self.accounts[plugin]) - - return self.plugins[plugin] - else: - return None - - def getAccountPlugins(self): - """ get all account instances""" - - plugins = [] - for plugin in self.accounts.keys(): - plugins.append(self.getAccountPlugin(plugin)) - - return plugins - #---------------------------------------------------------------------- - def loadAccounts(self): - """loads all accounts available""" - - if not exists("accounts.conf"): - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - - f = open("accounts.conf", "rb") - content = f.readlines() - version = content[0].split(":")[1].strip() if content else "" - f.close() - - if not version or int(version) < ACC_VERSION: - copy("accounts.conf", "accounts.backup") - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION)) - f.close() - self.core.log.warning(_("Account settings deleted, due to new config format.")) - return - - - - plugin = "" - name = "" - - for line in content[1:]: - line = line.strip() - - if not line: continue - if line.startswith("#"): continue - if line.startswith("version"): continue - - if line.endswith(":") and line.count(":") == 1: - plugin = line[:-1] - self.accounts[plugin] = {} - - elif line.startswith("@"): - option = line[1:].split() - self.accounts[plugin][name]["options"][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) - - elif ":" in line: - name, sep, pw = line.partition(":") - self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} - #---------------------------------------------------------------------- - def saveAccounts(self): - """save all account information""" - - f = open("accounts.conf", "wb") - f.write("version: " + str(ACC_VERSION) + "\n") - - for plugin, accounts in self.accounts.iteritems(): - f.write("\n") - f.write(plugin+":\n") - - for name,data in accounts.iteritems(): - f.write("\n\t%s:%s\n" % (name,data["password"]) ) - for option, values in data["options"].iteritems(): - f.write("\t@%s %s\n" % (option, " ".join(values))) - - f.close() - - - #---------------------------------------------------------------------- - def initAccountPlugins(self): - """init names""" - for name in self.core.pluginManager.getAccountPlugins(): - self.accounts[name] = {} - - #---------------------------------------------------------------------- - def updateAccount(self, plugin , user, password=None, options={}): - """add or update account""" - if self.accounts.has_key(plugin): - p = self.getAccountPlugin(plugin) - p.updateAccounts(user, password, options) - #since accounts is a ref in plugin self.accounts doesnt need to be updated here - - self.saveAccounts() - p.getAllAccounts(force=True) - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - - #---------------------------------------------------------------------- - def removeAccount(self, plugin, user): - """remove account""" - - if self.accounts.has_key(plugin): - p = self.getAccountPlugin(plugin) - p.removeAccount(user) - - self.saveAccounts() - p.getAllAccounts(force=True) - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - - - def getAccountInfos(self, force=True, refresh=False): - data = {} - - if refresh: - self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) - force = False - - for p in self.accounts.keys(): - if self.accounts[p]: - p = self.getAccountPlugin(p) - data[p.__name__] = p.getAllAccounts(force) - else: - data[p] = [] - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) - return data - - def sendChange(self): - e = AccountUpdateEvent() - self.core.pullManager.addEvent(e) diff --git a/module/PluginManager.py b/module/PluginManager.py deleted file mode 100644 index 3f11c067b..000000000 --- a/module/PluginManager.py +++ /dev/null @@ -1,329 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay, RaNaN -""" - -import re -import sys - -from os import listdir -from os import makedirs - -from os.path import isfile -from os.path import join -from os.path import exists -from os.path import abspath - -from sys import version_info -from itertools import chain - -try: - from ast import literal_eval -except ImportError: # python 2.5 - from module.SafeEval import safe_eval as literal_eval - - -IGNORE = ["FreakshareNet"] -#ignore this plugins in homefolder, add deleted plugins here - -class PluginManager(): - def __init__(self, core): - self.core = core - - #self.config = self.core.config - self.log = core.log - - self.crypterPlugins = {} - self.containerPlugins = {} - self.hosterPlugins = {} - self.captchaPlugins = {} - self.accountPlugins = {} - self.hookPlugins = {} - - self.createHomeDirs() - - self.createIndex() - - #@TODO plugin updater - #---------------------------------------------------------------------- - def createHomeDirs(self): - """create homedirectories containing plugins""" - #@TODO implement... - pass - - def createIndex(self): - """create information for all plugins available""" - - sys.path.append(abspath("")) - - if not exists("userplugins"): - makedirs("userplugins") - if not exists(join("userplugins", "__init__.py")): - f = open(join("userplugins", "__init__.py"), "wb") - f.close() - - self.rePattern = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') - self.reVersion = re.compile(r'__version__.*=.*("|\')([0-9.]+)') - self.reConfig = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) - - self.crypterPlugins = self.parse(_("Crypter"), "crypter", pattern=True) - self.containerPlugins = self.parse(_("Container"), "container", pattern=True) - self.hosterPlugins = self.parse(_("Hoster") ,"hoster", pattern=True) - - self.captchaPlugins = self.parse(_("Captcha"), "captcha") - self.accountPlugins = self.parse(_("Account"), "accounts", create=True) - self.hookPlugins = self.parse(_("Hook"), "hooks") - - self.log.debug(_("created index of plugins")) - - def parse(self, typ, folder, create=False, pattern=False, home={}): - """ - returns dict with information - home contains parsed plugins from module. - - { - name : {path, version, config, (pattern, re), (plugin, class)} - } - - """ - plugins = {} - if home: - pfolder = join("userplugins", folder) - if not exists(pfolder): - makedirs(pfolder) - if not exists(join(pfolder, "__init__.py")): - f = open(join(pfolder, "__init__.py"), "wb") - f.close() - - else: - pfolder = join(pypath, "module", "plugins", folder) - - for f in listdir(pfolder): - if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith("_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): - data = open(join(pfolder, f)) - content = data.read() - data.close() - - if f.endswith("_25.pyc") and not version_info[0:2] == (2, 5): - continue - elif f.endswith("_26.pyc") and not version_info[0:2] == (2, 6): - continue - elif f.endswith("_27.pyc") and not version_info[0:2] == (2, 7): - continue - - name = f[:-3] - if name[-1] == "." : name = name[:-4] - - - version = self.reVersion.findall(content) - if version: - version = float(version[0][1]) - else: - version = 0 - - if home and home.has_key(name): - if home[name]["v"] >= version: - continue - - - plugins[name] = {} - plugins[name]["v"] = version - - - module = f.replace(".pyc","").replace(".py","") - if home: - if name in IGNORE: - del plugins[name] - continue - path = "userplugins.%s.%s" % (folder, module) - else: - path = "module.plugins.%s.%s" % (folder, module) - - plugins[name]["name"] = module - plugins[name]["path"] = path - - - if pattern: - pattern = self.rePattern.findall(content) - - if pattern: - pattern = pattern[0][1] - else: - pattern = "^unmachtable$" - - plugins[name]["pattern"] = pattern - - try: - plugins[name]["re"] = re.compile(pattern) - except: - self.log.error(_("%s has invalid pattern.") % name) - - - config = self.reConfig.findall(content) - - if config: - config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) - if type(config[0]) == tuple: - config = [list(x) for x in config] - else: - config = [list(config)] - - if folder == "hooks": - config.append( ["load", "bool", "Load on startup", True if name not in ("XMPPInterface", "MultiHome") else False] ) - - for item in config: - self.core.config.addPluginConfig([name]+item) - - if not home: - temp = self.parse(typ, folder, create, pattern, plugins) - plugins.update(temp) - - return plugins - - #---------------------------------------------------------------------- - def parseUrls(self, urls): - """parse plugins for given list of urls""" - - last = None - res = [] # tupels of (url, plugin) - - for url in urls: - if type(url) not in (str, unicode, buffer): continue - found = False - - if last and last[1]["re"].match(url): - res.append((url, last[0])) - continue - - for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(), self.containerPlugins.iteritems() ): - if value["re"].match(url): - res.append((url, name)) - last = (name, value) - found = True - break - - if not found: - res.append((url, "BasePlugin")) - - return res - - #---------------------------------------------------------------------- - def getPlugin(self, name): - """return plugin module from hoster|decrypter|container""" - plugin = None - - if self.containerPlugins.has_key(name): - plugin = self.containerPlugins[name] - if self.crypterPlugins.has_key(name): - plugin = self.crypterPlugins[name] - if self.hosterPlugins.has_key(name): - plugin = self.hosterPlugins[name] - - - if plugin.has_key("module"): - return plugin["module"] - - plugin["module"] = __import__(plugin["path"], globals(), locals(), [plugin["name"]] , -1) - - return plugin["module"] - - - #---------------------------------------------------------------------- - def getCaptchaPlugin(self, name): - """return captcha modul if existent""" - if self.captchaPlugins.has_key(name): - plugin = self.captchaPlugins[name] - if plugin.has_key("class"): - return plugin["class"] - - module = __import__(plugin["path"], globals(), locals(), [plugin["name"]] , -1) - plugin["class"] = getattr(module, name) - - return plugin["class"] - - return None - #---------------------------------------------------------------------- - def getAccountPlugin(self, name): - """return account class if existent""" - if self.accountPlugins.has_key(name): - plugin = self.accountPlugins[name] - if plugin.has_key("class"): - return plugin["class"] - - module = __import__(plugin["path"], globals(), locals(), [plugin["name"]] , -1) - plugin["class"] = getattr(module, plugin["name"]) - - return plugin["class"] - - return None - - #---------------------------------------------------------------------- - def getAccountPlugins(self): - """return list of account plugin names""" - res = [] - - for name in self.accountPlugins.keys(): - res.append(name) - - return res - #---------------------------------------------------------------------- - def getHookPlugins(self): - """return list of hook classes""" - - classes = [] - - for name, value in self.hookPlugins.iteritems(): - if value.has_key("class"): - classes.append(value["class"]) - continue - - if not self.core.config.getPlugin(name, "load"): - continue - - try: - module = __import__(value["path"], globals(), locals(), [value["name"]] , -1) - except Exception, e: - self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e) }) - self.log.error(_("You should fix dependicies or deactivate load on startup.")) - continue - - pluginClass = getattr(module, name) - - value["class"] = pluginClass - - classes.append(pluginClass) - - return classes - - -if __name__ == "__main__": - _ = lambda x : x - pypath = "/home/christian/Projekte/pyload-0.4/module/plugins" - - from time import time - - p = PluginManager(None) - - a = time() - - test = [ "http://www.youtube.com/watch?v=%s" % x for x in range(0,100) ] - print p.parseUrls(test) - - b = time() - - print b-a ,"s" - diff --git a/module/RequestFactory.py b/module/RequestFactory.py deleted file mode 100644 index 5eaa32ed9..000000000 --- a/module/RequestFactory.py +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see . - - @author: mkaay -""" - -from threading import Lock -from module.network.Request import Request -from module.network.Browser import Browser -from module.network.XdccRequest import XdccRequest -from module.network.FtpRequest import FtpRequest -from time import time -from cookielib import CookieJar -from cookielib import Cookie - -class RequestFactory(): - def __init__(self, core): - self.lock = Lock() - self.core = core - self.requests = [] #seems useless - self.cookiejars = [] - self.iface = self.core.config["general"]["download_interface"] - - def getRequest(self, pluginName, account=None, type="HTTP"): - self.lock.acquire() - if type == "HTTP": - req = Browser() - if account: - cj = self.getCookieJar(pluginName, account) - req.setCookieJar(cj) - else: - req.setCookieJar(PyLoadCookieJar(pluginName)) - - elif type == "XDCC": - req = XdccRequest() - - elif type == "FTP": - req = FtpRequest() - - #self.requests.append((pluginName, account, req)) - self.lock.release() - return req - - def clean(self): - self.lock.acquire() - for req in self.requests: - req[2].clean() - self.lock.release() - - def getCookieJar(self, plugin, account=None): - for cj in self.cookiejars: - if (cj.plugin, cj.account) == (plugin, account): - return cj - cj = PyLoadCookieJar(plugin, account) - self.cookiejars.append(cj) - return cj - -class PyLoadCookieJar(CookieJar): - def __init__(self, plugin, account=None): - CookieJar.__init__(self) - self.cookies = {} - self.plugin = plugin - self.account = account - - def __del__(self): - if hasattr(self, "cookies"): - del self.cookies - if hasattr(self, "plugin"): - del self.plugin - - def getCookie(self, name): - print "getCookie not implemented!" - return None - - def setCookie(self, domain, name, value, path="/"): - c = Cookie(version=0, name=name, value=value, port=None, port_specified=False, - domain=domain, domain_specified=False, - domain_initial_dot=(domain.startswith(".")), path=path, path_specified=True, - secure=False, expires=None, discard=True, comment=None, - comment_url=None, rest={'HttpOnly': None}, rfc2109=False) - self.set_cookie(c) diff --git a/module/network/Browser.py b/module/network/Browser.py index df0f24b4d..3d2d5a73f 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -2,9 +2,9 @@ from random import randint from helper import * from os.path import join from logging import getLogger -from cookielib import CookieJar import zlib +from CookieJar import CookieJar from HTTPBase import HTTPBase from HTTPDownload import HTTPDownload from FTPBase import FTPDownload @@ -13,7 +13,7 @@ from XDCCBase import XDCCDownload from traceback import print_stack class Browser(object): - def __init__(self, interface=None, cookieJar=CookieJar(), bucket=None, proxies={}): + def __init__(self, interface=None, cookieJar=None, bucket=None, proxies={}): self.log = getLogger("log") self.lastURL = None @@ -21,7 +21,7 @@ class Browser(object): self.bucket = bucket self.http = HTTPBase(interface=interface, proxies=proxies) - self.setCookieJar(cookieJar) + self.setCookieJar(cookieJar if cookieJar else CookieJar()) self.proxies = proxies def setCookieJar(self, cookieJar): diff --git a/module/network/CookieJar.py b/module/network/CookieJar.py new file mode 100644 index 000000000..fc6b5e076 --- /dev/null +++ b/module/network/CookieJar.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay, RaNaN +""" + +from cookielib import CookieJar as PyCookieJar +from cookielib import Cookie + + +class CookieJar(PyCookieJar): + def __init__(self, pluginName=None, account=None): + PyCookieJar.__init__(self) + self.plugin = pluginName + self.account = account + + + def getCookie(self, name): + print "getCookie not implemented!" + return None + + def setCookie(self, domain, name, value, path="/"): + c = Cookie(version=0, name=name, value=value, port=None, port_specified=False, + domain=domain, domain_specified=False, + domain_initial_dot=(domain.startswith(".")), path=path, path_specified=True, + secure=False, expires=None, discard=True, comment=None, + comment_url=None, rest={'HttpOnly': None}, rfc2109=False) + self.set_cookie(c) \ No newline at end of file diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py new file mode 100644 index 000000000..d73ab5e20 --- /dev/null +++ b/module/network/RequestFactory.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay, RaNaN +""" + +from threading import Lock + +from Browser import Browser +from HTTPBase import HTTPBase +from CookieJar import CookieJar + +class RequestFactory(): + def __init__(self, core): + self.lock = Lock() + self.core = core + self.cookiejars = {} + + iface = property(lambda self: self.core.config["general"]["download_interface"]) + + def getRequest(self, pluginName, account=None): + self.lock.acquire() + + req = Browser() + #@TODO proxy stuff, bucket + + if account: + cj = self.getCookieJar(pluginName, account) + req.setCookieJar(cj) + else: + req.setCookieJar(CookieJar(pluginName)) + + self.lock.release() + return req + + def getURL(self, url): + base = HTTPBase() + #@TODO proxies + #@TODO post,get... + + resp = base.getResponse(url) + resp = resp.read() + return resp + + def getCookieJar(self, pluginName, account=None): + if self.cookiejars.has_key((pluginName, account)): + return self.cookiejars[(pluginName, account)] + + cj = CookieJar(pluginName, account) + self.cookiejars[(pluginName, account)] = cj + return cj + +# needs pyreq in global namespace +def getURL(url): + pyreq.getURL(url) \ No newline at end of file diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py new file mode 100644 index 000000000..ab35334fc --- /dev/null +++ b/module/plugins/AccountManager.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from os.path import exists +from shutil import copy + +from module.PullEvents import AccountUpdateEvent + +ACC_VERSION = 1 + +######################################################################## +class AccountManager(): + """manages all accounts""" + + #---------------------------------------------------------------------- + def __init__(self, core): + """Constructor""" + + self.core = core + + self.accounts = {} # key = ( plugin ) + self.plugins = {} + + self.initAccountPlugins() + + self.loadAccounts() + + self.saveAccounts() # save to add categories to conf + + #---------------------------------------------------------------------- + def getAccountPlugin(self, plugin): + """get account instance for plugin or None if anonymous""" + if self.accounts.has_key(plugin): + if not self.plugins.has_key(plugin): + self.plugins[plugin] = self.core.pluginManager.getAccountPlugin(plugin)(self, self.accounts[plugin]) + + return self.plugins[plugin] + else: + return None + + def getAccountPlugins(self): + """ get all account instances""" + + plugins = [] + for plugin in self.accounts.keys(): + plugins.append(self.getAccountPlugin(plugin)) + + return plugins + #---------------------------------------------------------------------- + def loadAccounts(self): + """loads all accounts available""" + + if not exists("accounts.conf"): + f = open("accounts.conf", "wb") + f.write("version: " + str(ACC_VERSION)) + f.close() + + f = open("accounts.conf", "rb") + content = f.readlines() + version = content[0].split(":")[1].strip() if content else "" + f.close() + + if not version or int(version) < ACC_VERSION: + copy("accounts.conf", "accounts.backup") + f = open("accounts.conf", "wb") + f.write("version: " + str(ACC_VERSION)) + f.close() + self.core.log.warning(_("Account settings deleted, due to new config format.")) + return + + + + plugin = "" + name = "" + + for line in content[1:]: + line = line.strip() + + if not line: continue + if line.startswith("#"): continue + if line.startswith("version"): continue + + if line.endswith(":") and line.count(":") == 1: + plugin = line[:-1] + self.accounts[plugin] = {} + + elif line.startswith("@"): + option = line[1:].split() + self.accounts[plugin][name]["options"][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:]) + + elif ":" in line: + name, sep, pw = line.partition(":") + self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True} + #---------------------------------------------------------------------- + def saveAccounts(self): + """save all account information""" + + f = open("accounts.conf", "wb") + f.write("version: " + str(ACC_VERSION) + "\n") + + for plugin, accounts in self.accounts.iteritems(): + f.write("\n") + f.write(plugin+":\n") + + for name,data in accounts.iteritems(): + f.write("\n\t%s:%s\n" % (name,data["password"]) ) + for option, values in data["options"].iteritems(): + f.write("\t@%s %s\n" % (option, " ".join(values))) + + f.close() + + + #---------------------------------------------------------------------- + def initAccountPlugins(self): + """init names""" + for name in self.core.pluginManager.getAccountPlugins(): + self.accounts[name] = {} + + #---------------------------------------------------------------------- + def updateAccount(self, plugin , user, password=None, options={}): + """add or update account""" + if self.accounts.has_key(plugin): + p = self.getAccountPlugin(plugin) + p.updateAccounts(user, password, options) + #since accounts is a ref in plugin self.accounts doesnt need to be updated here + + self.saveAccounts() + p.getAllAccounts(force=True) + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + + #---------------------------------------------------------------------- + def removeAccount(self, plugin, user): + """remove account""" + + if self.accounts.has_key(plugin): + p = self.getAccountPlugin(plugin) + p.removeAccount(user) + + self.saveAccounts() + p.getAllAccounts(force=True) + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + + + def getAccountInfos(self, force=True, refresh=False): + data = {} + + if refresh: + self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos) + force = False + + for p in self.accounts.keys(): + if self.accounts[p]: + p = self.getAccountPlugin(p) + data[p.__name__] = p.getAllAccounts(force) + else: + data[p] = [] + e = AccountUpdateEvent() + self.core.pullManager.addEvent(e) + return data + + def sendChange(self): + e = AccountUpdateEvent() + self.core.pullManager.addEvent(e) diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py new file mode 100644 index 000000000..3f11c067b --- /dev/null +++ b/module/plugins/PluginManager.py @@ -0,0 +1,329 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: mkaay, RaNaN +""" + +import re +import sys + +from os import listdir +from os import makedirs + +from os.path import isfile +from os.path import join +from os.path import exists +from os.path import abspath + +from sys import version_info +from itertools import chain + +try: + from ast import literal_eval +except ImportError: # python 2.5 + from module.SafeEval import safe_eval as literal_eval + + +IGNORE = ["FreakshareNet"] +#ignore this plugins in homefolder, add deleted plugins here + +class PluginManager(): + def __init__(self, core): + self.core = core + + #self.config = self.core.config + self.log = core.log + + self.crypterPlugins = {} + self.containerPlugins = {} + self.hosterPlugins = {} + self.captchaPlugins = {} + self.accountPlugins = {} + self.hookPlugins = {} + + self.createHomeDirs() + + self.createIndex() + + #@TODO plugin updater + #---------------------------------------------------------------------- + def createHomeDirs(self): + """create homedirectories containing plugins""" + #@TODO implement... + pass + + def createIndex(self): + """create information for all plugins available""" + + sys.path.append(abspath("")) + + if not exists("userplugins"): + makedirs("userplugins") + if not exists(join("userplugins", "__init__.py")): + f = open(join("userplugins", "__init__.py"), "wb") + f.close() + + self.rePattern = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)') + self.reVersion = re.compile(r'__version__.*=.*("|\')([0-9.]+)') + self.reConfig = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE) + + self.crypterPlugins = self.parse(_("Crypter"), "crypter", pattern=True) + self.containerPlugins = self.parse(_("Container"), "container", pattern=True) + self.hosterPlugins = self.parse(_("Hoster") ,"hoster", pattern=True) + + self.captchaPlugins = self.parse(_("Captcha"), "captcha") + self.accountPlugins = self.parse(_("Account"), "accounts", create=True) + self.hookPlugins = self.parse(_("Hook"), "hooks") + + self.log.debug(_("created index of plugins")) + + def parse(self, typ, folder, create=False, pattern=False, home={}): + """ + returns dict with information + home contains parsed plugins from module. + + { + name : {path, version, config, (pattern, re), (plugin, class)} + } + + """ + plugins = {} + if home: + pfolder = join("userplugins", folder) + if not exists(pfolder): + makedirs(pfolder) + if not exists(join(pfolder, "__init__.py")): + f = open(join(pfolder, "__init__.py"), "wb") + f.close() + + else: + pfolder = join(pypath, "module", "plugins", folder) + + for f in listdir(pfolder): + if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith("_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"): + data = open(join(pfolder, f)) + content = data.read() + data.close() + + if f.endswith("_25.pyc") and not version_info[0:2] == (2, 5): + continue + elif f.endswith("_26.pyc") and not version_info[0:2] == (2, 6): + continue + elif f.endswith("_27.pyc") and not version_info[0:2] == (2, 7): + continue + + name = f[:-3] + if name[-1] == "." : name = name[:-4] + + + version = self.reVersion.findall(content) + if version: + version = float(version[0][1]) + else: + version = 0 + + if home and home.has_key(name): + if home[name]["v"] >= version: + continue + + + plugins[name] = {} + plugins[name]["v"] = version + + + module = f.replace(".pyc","").replace(".py","") + if home: + if name in IGNORE: + del plugins[name] + continue + path = "userplugins.%s.%s" % (folder, module) + else: + path = "module.plugins.%s.%s" % (folder, module) + + plugins[name]["name"] = module + plugins[name]["path"] = path + + + if pattern: + pattern = self.rePattern.findall(content) + + if pattern: + pattern = pattern[0][1] + else: + pattern = "^unmachtable$" + + plugins[name]["pattern"] = pattern + + try: + plugins[name]["re"] = re.compile(pattern) + except: + self.log.error(_("%s has invalid pattern.") % name) + + + config = self.reConfig.findall(content) + + if config: + config = literal_eval(config[0].strip().replace("\n", "").replace("\r", "")) + if type(config[0]) == tuple: + config = [list(x) for x in config] + else: + config = [list(config)] + + if folder == "hooks": + config.append( ["load", "bool", "Load on startup", True if name not in ("XMPPInterface", "MultiHome") else False] ) + + for item in config: + self.core.config.addPluginConfig([name]+item) + + if not home: + temp = self.parse(typ, folder, create, pattern, plugins) + plugins.update(temp) + + return plugins + + #---------------------------------------------------------------------- + def parseUrls(self, urls): + """parse plugins for given list of urls""" + + last = None + res = [] # tupels of (url, plugin) + + for url in urls: + if type(url) not in (str, unicode, buffer): continue + found = False + + if last and last[1]["re"].match(url): + res.append((url, last[0])) + continue + + for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(), self.containerPlugins.iteritems() ): + if value["re"].match(url): + res.append((url, name)) + last = (name, value) + found = True + break + + if not found: + res.append((url, "BasePlugin")) + + return res + + #---------------------------------------------------------------------- + def getPlugin(self, name): + """return plugin module from hoster|decrypter|container""" + plugin = None + + if self.containerPlugins.has_key(name): + plugin = self.containerPlugins[name] + if self.crypterPlugins.has_key(name): + plugin = self.crypterPlugins[name] + if self.hosterPlugins.has_key(name): + plugin = self.hosterPlugins[name] + + + if plugin.has_key("module"): + return plugin["module"] + + plugin["module"] = __import__(plugin["path"], globals(), locals(), [plugin["name"]] , -1) + + return plugin["module"] + + + #---------------------------------------------------------------------- + def getCaptchaPlugin(self, name): + """return captcha modul if existent""" + if self.captchaPlugins.has_key(name): + plugin = self.captchaPlugins[name] + if plugin.has_key("class"): + return plugin["class"] + + module = __import__(plugin["path"], globals(), locals(), [plugin["name"]] , -1) + plugin["class"] = getattr(module, name) + + return plugin["class"] + + return None + #---------------------------------------------------------------------- + def getAccountPlugin(self, name): + """return account class if existent""" + if self.accountPlugins.has_key(name): + plugin = self.accountPlugins[name] + if plugin.has_key("class"): + return plugin["class"] + + module = __import__(plugin["path"], globals(), locals(), [plugin["name"]] , -1) + plugin["class"] = getattr(module, plugin["name"]) + + return plugin["class"] + + return None + + #---------------------------------------------------------------------- + def getAccountPlugins(self): + """return list of account plugin names""" + res = [] + + for name in self.accountPlugins.keys(): + res.append(name) + + return res + #---------------------------------------------------------------------- + def getHookPlugins(self): + """return list of hook classes""" + + classes = [] + + for name, value in self.hookPlugins.iteritems(): + if value.has_key("class"): + classes.append(value["class"]) + continue + + if not self.core.config.getPlugin(name, "load"): + continue + + try: + module = __import__(value["path"], globals(), locals(), [value["name"]] , -1) + except Exception, e: + self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e) }) + self.log.error(_("You should fix dependicies or deactivate load on startup.")) + continue + + pluginClass = getattr(module, name) + + value["class"] = pluginClass + + classes.append(pluginClass) + + return classes + + +if __name__ == "__main__": + _ = lambda x : x + pypath = "/home/christian/Projekte/pyload-0.4/module/plugins" + + from time import time + + p = PluginManager(None) + + a = time() + + test = [ "http://www.youtube.com/watch?v=%s" % x for x in range(0,100) ] + print p.parseUrls(test) + + b = time() + + print b-a ,"s" + diff --git a/pyLoadCore.py b/pyLoadCore.py index 805270c59..bdbd29215 100755 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -22,6 +22,8 @@ """ CURRENT_VERSION = '0.4.3' +import __builtin__ + from getopt import GetoptError from getopt import getopt import gettext @@ -51,14 +53,14 @@ from traceback import print_exc from xmlrpclib import Binary from module import InitHomeDir -from module.AccountManager import AccountManager +from module.plugins.AccountManager import AccountManager from module.CaptchaManager import CaptchaManager from module.ConfigParser import ConfigParser from module.FileDatabase import FileHandler from module.HookManager import HookManager -from module.PluginManager import PluginManager +from module.plugins.PluginManager import PluginManager from module.PullEvents import PullManager -from module.RequestFactory import RequestFactory +from module.network.RequestFactory import RequestFactory from module.ThreadManager import ThreadManager import module.remote.SecureXMLRPCServer as Server from module.web.ServerThread import WebServer @@ -313,6 +315,7 @@ class Core(object): self.requestFactory = RequestFactory(self) + __builtin__.pyreq = self.requestFactory #path.append(self.plugin_folder) -- cgit v1.2.3