summaryrefslogtreecommitdiffstats
path: root/module/plugins
diff options
context:
space:
mode:
authorGravatar Walter Purcaro <vuolter@gmail.com> 2014-11-22 19:38:25 +0100
committerGravatar Walter Purcaro <vuolter@gmail.com> 2014-11-22 19:38:25 +0100
commitef4bc4b73756565e40c7453f6b71bc1021735033 (patch)
treede307e0b2d9efd496f942b09d6be6dc4a7dc4850 /module/plugins
parent[SimpleHoster] Fix statusMap generator for python 2.5 (diff)
downloadpyload-ef4bc4b73756565e40c7453f6b71bc1021735033.tar.xz
Revert plugins to stable
Diffstat (limited to 'module/plugins')
-rw-r--r--module/plugins/Account.py159
-rw-r--r--module/plugins/AccountManager.py152
-rw-r--r--module/plugins/Container.py71
-rw-r--r--module/plugins/Crypter.py133
-rw-r--r--module/plugins/Hook.py87
-rw-r--r--module/plugins/Hoster.py32
-rw-r--r--module/plugins/Plugin.py376
-rw-r--r--module/plugins/PluginManager.py263
8 files changed, 553 insertions, 720 deletions
diff --git a/module/plugins/Account.py b/module/plugins/Account.py
index 2f7c5d30d..c147404e0 100644
--- a/module/plugins/Account.py
+++ b/module/plugins/Account.py
@@ -1,14 +1,30 @@
# -*- coding: utf-8 -*-
+"""
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License,
+ or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+ @author: mkaay
+"""
+
from random import choice
from time import time
from traceback import print_exc
from threading import RLock
-from module.plugins.Plugin import Base
+from Plugin import Base
from module.utils import compare_time, parseFileSize, lock
-
class WrongPassword(Exception):
pass
@@ -19,19 +35,17 @@ class Account(Base):
Just overwrite `login` and cookies will be stored and account becomes accessible in\
associated hoster plugin. Plugin should also provide `loadAccountInfo`
"""
- __name__ = "Account"
- __type__ = "account"
- __version__ = "0.03"
-
- __description__ = """Base account plugin"""
- __license__ = "GPLv3"
- __authors__ = [("mkaay", "mkaay@mkaay.de")]
+ __name__ = "Account"
+ __version__ = "0.2"
+ __type__ = "account"
+ __description__ = """Account Plugin"""
+ __author_name__ = ("mkaay")
+ __author_mail__ = ("mkaay@mkaay.de")
-
- #: after that time (in minutes) pyload will relogin the account
- login_timeout = 10 * 60
- #: after that time (in minutes) account data will be reloaded
- info_threshold = 10 * 60
+ #: after that time [in minutes] pyload will relogin the account
+ login_timeout = 600
+ #: account data will be reloaded after this time
+ info_threshold = 600
def __init__(self, manager, accounts):
@@ -39,19 +53,16 @@ class Account(Base):
self.manager = manager
self.accounts = {}
- self.infos = {} #: cache for account information
+ self.infos = {} # cache for account information
self.lock = RLock()
- self.timestamps = {}
-
- self.init()
+ self.timestamps = {}
self.setAccounts(accounts)
-
+ self.init()
def init(self):
pass
-
def login(self, user, data, req):
"""login into account, the cookies will be saved so user can be recognized
@@ -61,34 +72,29 @@ class Account(Base):
"""
pass
-
@lock
def _login(self, user, data):
# set timestamp for login
self.timestamps[user] = time()
-
+
req = self.getAccountRequest(user)
try:
self.login(user, data, req)
except WrongPassword:
self.logWarning(
- _("Could not login with account %(user)s | %(msg)s") % {"user": user,
- "msg": _("Wrong Password")})
- success = data['valid'] = False
+ _("Could not login with account %(user)s | %(msg)s") % {"user": user
+ , "msg": _("Wrong Password")})
+ data["valid"] = False
+
except Exception, e:
self.logWarning(
- _("Could not login with account %(user)s | %(msg)s") % {"user": user,
- "msg": e})
- success = data['valid'] = False
+ _("Could not login with account %(user)s | %(msg)s") % {"user": user
+ , "msg": e})
+ data["valid"] = False
if self.core.debug:
print_exc()
- else:
- success = True
finally:
- if req:
- req.close()
- return success
-
+ if req: req.close()
def relogin(self, user):
req = self.getAccountRequest(user)
@@ -98,8 +104,7 @@ class Account(Base):
if user in self.infos:
del self.infos[user] #delete old information
- return self._login(user, self.accounts[user])
-
+ self._login(user, self.accounts[user])
def setAccounts(self, accounts):
self.accounts = accounts
@@ -107,26 +112,24 @@ class Account(Base):
self._login(user, data)
self.infos[user] = {}
-
def updateAccounts(self, user, password=None, options={}):
""" updates account and return true if anything changed """
if user in self.accounts:
- self.accounts[user]['valid'] = True #do not remove or accounts will not login
+ self.accounts[user]["valid"] = True #do not remove or accounts will not login
if password:
- self.accounts[user]['password'] = password
+ self.accounts[user]["password"] = password
self.relogin(user)
return True
if options:
- before = self.accounts[user]['options']
- self.accounts[user]['options'].update(options)
- return self.accounts[user]['options'] != before
+ before = self.accounts[user]["options"]
+ self.accounts[user]["options"].update(options)
+ return self.accounts[user]["options"] != before
else:
self.accounts[user] = {"password": password, "options": options, "valid": True}
self._login(user, self.accounts[user])
return True
-
def removeAccount(self, user):
if user in self.accounts:
del self.accounts[user]
@@ -135,7 +138,6 @@ class Account(Base):
if user in self.timestamps:
del self.timestamps[user]
-
@lock
def getAccountInfo(self, name, force=False):
"""retrieve account infos for an user, do **not** overwrite this method!\\
@@ -157,14 +159,12 @@ class Account(Base):
raise Exception("Wrong return format")
except Exception, e:
infos = {"error": str(e)}
- print_exc()
- if req:
- req.close()
+ if req: req.close()
- self.logDebug("Account Info: %s" % infos)
+ self.logDebug("Account Info: %s" % str(infos))
- infos['timestamp'] = time()
+ infos["timestamp"] = time()
self.infos[name] = infos
elif "timestamp" in self.infos[name] and self.infos[name][
"timestamp"] + self.info_threshold * 60 < time():
@@ -174,11 +174,9 @@ class Account(Base):
data.update(self.infos[name])
return data
-
def isPremium(self, user):
info = self.getAccountInfo(user)
- return info['premium']
-
+ return info["premium"]
def loadAccountInfo(self, name, req=None):
"""this should be overwritten in account plugin,\
@@ -188,22 +186,22 @@ class Account(Base):
:param req: `Request` instance
:return:
"""
- return {"validuntil": None, #: -1 for unlimited
- "login": name,
- # "password": self.accounts[name]['password'], #: commented due security reason
- "options": self.accounts[name]['options'],
- "valid": self.accounts[name]['valid'],
- "trafficleft": None, #: in kb, -1 for unlimited
- "maxtraffic": None,
- "premium": None,
- "timestamp": 0, #: time this info was retrieved
- "type": self.__name__}
-
+ return {
+ "validuntil": None, # -1 for unlimited
+ "login": name,
+ #"password": self.accounts[name]["password"], #@XXX: security
+ "options": self.accounts[name]["options"],
+ "valid": self.accounts[name]["valid"],
+ "trafficleft": None, # in kb, -1 for unlimited
+ "maxtraffic": None,
+ "premium": True, #useful for free accounts
+ "timestamp": 0, #time this info was retrieved
+ "type": self.__name__,
+ }
def getAllAccounts(self, force=False):
return [self.getAccountInfo(user, force) for user, data in self.accounts.iteritems()]
-
def getAccountRequest(self, user=None):
if not user:
user, data = self.selectAccount()
@@ -213,7 +211,6 @@ class Account(Base):
req = self.core.requestFactory.getRequest(self.__name__, user)
return req
-
def getAccountCookies(self, user=None):
if not user:
user, data = self.selectAccount()
@@ -223,21 +220,19 @@ class Account(Base):
cj = self.core.requestFactory.getCookieJar(self.__name__, user)
return cj
-
def getAccountData(self, user):
return self.accounts[user]
-
def selectAccount(self):
""" returns an valid account name and data"""
usable = []
for user, data in self.accounts.iteritems():
- if not data['valid']: continue
+ if not data["valid"]: continue
- if "time" in data['options'] and data['options']['time']:
+ if "time" in data["options"] and data["options"]["time"]:
time_data = ""
try:
- time_data = data['options']['time'][0]
+ time_data = data["options"]["time"][0]
start, end = time_data.split("-")
if not compare_time(start.split(":"), end.split(":")):
continue
@@ -246,10 +241,10 @@ class Account(Base):
if user in self.infos:
if "validuntil" in self.infos[user]:
- if self.infos[user]['validuntil'] > 0 and time() > self.infos[user]['validuntil']:
+ if self.infos[user]["validuntil"] > 0 and time() > self.infos[user]["validuntil"]:
continue
if "trafficleft" in self.infos[user]:
- if self.infos[user]['trafficleft'] == 0:
+ if self.infos[user]["trafficleft"] == 0:
continue
usable.append((user, data))
@@ -257,19 +252,15 @@ class Account(Base):
if not usable: return None, None
return choice(usable)
-
def canUse(self):
return False if self.selectAccount() == (None, None) else True
-
def parseTraffic(self, string): #returns kbyte
- return parseFileSize(string)
-
+ return parseFileSize(string) / 1024
def wrongPassword(self):
raise WrongPassword
-
def empty(self, user):
if user in self.infos:
self.logWarning(_("Account %s has not enough traffic, checking again in 30min") % user)
@@ -277,7 +268,6 @@ class Account(Base):
self.infos[user].update({"trafficleft": 0})
self.scheduleRefresh(user, 30 * 60)
-
def expired(self, user):
if user in self.infos:
self.logWarning(_("Account %s is expired, checking again in 1h") % user)
@@ -285,21 +275,18 @@ class Account(Base):
self.infos[user].update({"validuntil": time() - 1})
self.scheduleRefresh(user, 60 * 60)
-
def scheduleRefresh(self, user, time=0, force=True):
""" add task to refresh account info to sheduler """
self.logDebug("Scheduled Account refresh for %s in %s seconds." % (user, time))
self.core.scheduler.addJob(time, self.getAccountInfo, [user, force])
-
@lock
def checkLogin(self, user):
""" checks if user is still logged in """
if user in self.timestamps:
- if self.login_timeout > 0 and self.timestamps[user] + self.login_timeout * 60 < time():
+ if self.timestamps[user] + self.login_timeout * 60 < time():
self.logDebug("Reached login timeout for %s" % user)
- return self.relogin(user)
- else:
- return True
- else:
- return False
+ self.relogin(user)
+ return False
+
+ return True
diff --git a/module/plugins/AccountManager.py b/module/plugins/AccountManager.py
index 3cb14403a..fc521d36c 100644
--- a/module/plugins/AccountManager.py
+++ b/module/plugins/AccountManager.py
@@ -1,4 +1,21 @@
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
+"""
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License,
+ or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+ @author: RaNaN
+"""
from os.path import exists
from shutil import copy
@@ -10,11 +27,9 @@ from module.utils import chmod, lock
ACC_VERSION = 1
-
class AccountManager():
"""manages all accounts"""
-
#----------------------------------------------------------------------
def __init__(self, core):
"""Constructor"""
@@ -25,7 +40,6 @@ class AccountManager():
self.initPlugins()
self.saveAccounts() # save to add categories to conf
-
def initPlugins(self):
self.accounts = {} # key = ( plugin )
self.plugins = {}
@@ -36,110 +50,97 @@ class AccountManager():
def getAccountPlugin(self, plugin):
"""get account instance for plugin or None if anonymous"""
- try:
- if plugin in self.accounts:
- if plugin not in self.plugins:
- klass = self.core.pluginManager.loadClass("accounts", plugin)
- if klass:
- self.plugins[plugin] = klass(self, self.accounts[plugin])
- else:
- raise
-
- return self.plugins[plugin]
- else:
- raise
- except:
- return None
+ if plugin in self.accounts:
+ if plugin not in self.plugins:
+ self.plugins[plugin] = self.core.pluginManager.loadClass("accounts", plugin)(self, self.accounts[plugin])
+ return self.plugins[plugin]
+ else:
+ return None
def getAccountPlugins(self):
""" get all account instances"""
-
+
plugins = []
for plugin in self.accounts.keys():
plugins.append(self.getAccountPlugin(plugin))
-
+
return plugins
-
-
#----------------------------------------------------------------------
def loadAccounts(self):
"""loads all accounts available"""
-
- try:
- with open("accounts.conf", "a+") as f:
- content = f.readlines()
- version = content[0].split(":")[1].strip() if content else ""
-
- if not version or int(version) < ACC_VERSION:
- copy("accounts.conf", "accounts.backup")
- f.seek(0)
- f.write("version: " + str(ACC_VERSION))
-
- self.core.log.warning(_("Account settings deleted, due to new config format"))
- return
-
- except IOError, e:
- self.logError(e)
+
+ if not exists("accounts.conf"):
+ f = open("accounts.conf", "wb")
+ f.write("version: " + str(ACC_VERSION))
+ f.close()
+
+ f = open("accounts.conf", "rb")
+ content = f.readlines()
+ version = content[0].split(":")[1].strip() if content else ""
+ f.close()
+
+ if not version or int(version) < ACC_VERSION:
+ copy("accounts.conf", "accounts.backup")
+ f = open("accounts.conf", "wb")
+ f.write("version: " + str(ACC_VERSION))
+ f.close()
+ self.core.log.warning(_("Account settings deleted, due to new config format."))
return
-
+
+
+
plugin = ""
name = ""
-
+
for line in content[1:]:
line = line.strip()
-
+
if not line: continue
if line.startswith("#"): continue
if line.startswith("version"): continue
-
+
if line.endswith(":") and line.count(":") == 1:
plugin = line[:-1]
self.accounts[plugin] = {}
-
+
elif line.startswith("@"):
try:
option = line[1:].split()
- self.accounts[plugin][name]['options'][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:])
+ self.accounts[plugin][name]["options"][option[0]] = [] if len(option) < 2 else ([option[1]] if len(option) < 3 else option[1:])
except:
pass
-
+
elif ":" in line:
name, sep, pw = line.partition(":")
self.accounts[plugin][name] = {"password": pw, "options": {}, "valid": True}
-
-
#----------------------------------------------------------------------
def saveAccounts(self):
"""save all account information"""
-
- try:
- with open("accounts.conf", "wb") as f:
- f.write("version: " + str(ACC_VERSION) + "\n")
-
- for plugin, accounts in self.accounts.iteritems():
- f.write("\n")
- f.write(plugin+":\n")
-
- for name,data in accounts.iteritems():
- f.write("\n\t%s:%s\n" % (name,data['password']) )
- if data['options']:
- for option, values in data['options'].iteritems():
- f.write("\t@%s %s\n" % (option, " ".join(values)))
-
- chmod(f.name, 0600)
-
- except Exception, e:
- self.logError(e)
-
-
+
+ f = open("accounts.conf", "wb")
+ f.write("version: " + str(ACC_VERSION) + "\n")
+
+ for plugin, accounts in self.accounts.iteritems():
+ f.write("\n")
+ f.write(plugin+":\n")
+
+ for name,data in accounts.iteritems():
+ f.write("\n\t%s:%s\n" % (name,data["password"]) )
+ if data["options"]:
+ for option, values in data["options"].iteritems():
+ f.write("\t@%s %s\n" % (option, " ".join(values)))
+
+ f.close()
+ chmod(f.name, 0600)
+
+
#----------------------------------------------------------------------
def initAccountPlugins(self):
"""init names"""
for name in self.core.pluginManager.getAccountPlugins():
self.accounts[name] = {}
-
-
+
@lock
def updateAccount(self, plugin , user, password=None, options={}):
"""add or update account"""
@@ -147,22 +148,20 @@ class AccountManager():
p = self.getAccountPlugin(plugin)
updated = p.updateAccounts(user, password, options)
#since accounts is a ref in plugin self.accounts doesnt need to be updated here
-
+
self.saveAccounts()
if updated: p.scheduleRefresh(user, force=False)
-
-
+
@lock
def removeAccount(self, plugin, user):
"""remove account"""
-
+
if plugin in self.accounts:
p = self.getAccountPlugin(plugin)
p.removeAccount(user)
self.saveAccounts()
-
@lock
def getAccountInfos(self, force=True, refresh=False):
data = {}
@@ -170,7 +169,7 @@ class AccountManager():
if refresh:
self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos)
force = False
-
+
for p in self.accounts.keys():
if self.accounts[p]:
p = self.getAccountPlugin(p)
@@ -180,8 +179,7 @@ class AccountManager():
e = AccountUpdateEvent()
self.core.pullManager.addEvent(e)
return data
-
-
+
def sendChange(self):
e = AccountUpdateEvent()
self.core.pullManager.addEvent(e)
diff --git a/module/plugins/Container.py b/module/plugins/Container.py
index 913c80481..c233d3710 100644
--- a/module/plugins/Container.py
+++ b/module/plugins/Container.py
@@ -1,25 +1,36 @@
# -*- coding: utf-8 -*-
-import re
-
-from os import remove
-from os.path import basename, exists
+"""
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License,
+ or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+ @author: mkaay
+"""
from module.plugins.Crypter import Crypter
-from module.utils import save_join
+from os.path import join, exists, basename
+from os import remove
+import re
class Container(Crypter):
- __name__ = "Container"
- __type__ = "container"
- __version__ = "0.01"
-
+ __name__ = "Container"
+ __version__ = "0.1"
__pattern__ = None
- __config__ = [] #: [("name", "type", "desc", "default")]
-
- __description__ = """Base container decrypter plugin"""
- __license__ = "GPLv3"
- __authors__ = [("mkaay", "mkaay@mkaay.de")]
+ __type__ = "container"
+ __description__ = """Base container plugin"""
+ __author_name__ = ("mkaay")
+ __author_mail__ = ("mkaay@mkaay.de")
def preprocessing(self, thread):
@@ -27,38 +38,38 @@ class Container(Crypter):
self.setup()
self.thread = thread
-
+
self.loadToDisk()
self.decrypt(self.pyfile)
self.deleteTmp()
-
+
self.createPackages()
-
+
def loadToDisk(self):
- """loads container to disk if its stored remotely and overwrite url,
+ """loads container to disk if its stored remotely and overwrite url,
or check existent on several places at disk"""
-
+
if self.pyfile.url.startswith("http"):
self.pyfile.name = re.findall("([^\/=]+)", self.pyfile.url)[-1]
content = self.load(self.pyfile.url)
- self.pyfile.url = save_join(self.config['general']['download_folder'], self.pyfile.name)
- try:
- with open(self.pyfile.url, "wb") as f:
- f.write(content)
- except IOError, e:
- self.fail(str(e))
-
+ self.pyfile.url = join(self.config["general"]["download_folder"], self.pyfile.name)
+ f = open(self.pyfile.url, "wb" )
+ f.write(content)
+ f.close()
+
else:
self.pyfile.name = basename(self.pyfile.url)
if not exists(self.pyfile.url):
- if exists(save_join(pypath, self.pyfile.url)):
- self.pyfile.url = save_join(pypath, self.pyfile.url)
+ if exists(join(pypath, self.pyfile.url)):
+ self.pyfile.url = join(pypath, self.pyfile.url)
else:
- self.fail(_("File not exists"))
-
+ self.fail(_("File not exists."))
+
def deleteTmp(self):
if self.pyfile.name.startswith("tmp_"):
remove(self.pyfile.url)
+
+
diff --git a/module/plugins/Crypter.py b/module/plugins/Crypter.py
index 8888dc4e6..d1549fe80 100644
--- a/module/plugins/Crypter.py
+++ b/module/plugins/Crypter.py
@@ -1,107 +1,72 @@
# -*- coding: utf-8 -*-
-from module.plugins.Plugin import Plugin
-from module.utils import save_path
+"""
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License,
+ or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+ @author: mkaay
+"""
+from module.plugins.Plugin import Plugin
class Crypter(Plugin):
- __name__ = "Crypter"
- __type__ = "crypter"
- __version__ = "0.05"
-
+ __name__ = "Crypter"
+ __version__ = "0.1"
__pattern__ = None
- __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package']
- ("subfolder_per_package", "bool", "Create a subfolder for each package", True)]
-
- __description__ = """Base decrypter plugin"""
- __license__ = "GPLv3"
- __authors__ = [("Walter Purcaro", "vuolter@gmail.com")]
-
-
- html = None #: last html loaded
-
-
+ __type__ = "container"
+ __description__ = """Base crypter plugin"""
+ __author_name__ = ("mkaay")
+ __author_mail__ = ("mkaay@mkaay.de")
+
def __init__(self, pyfile):
+ Plugin.__init__(self, pyfile)
+
#: Put all packages here. It's a list of tuples like: ( name, [list of links], folder )
self.packages = []
#: List of urls, pyLoad will generate packagenames
self.urls = []
-
- Plugin.__init__(self, pyfile)
-
-
- def process(self, pyfile):
- """ main method """
-
- self.decrypt(pyfile)
-
- if self.urls:
- self.generatePackages()
-
- elif not self.packages:
- self.error(_("No link extracted"), "decrypt")
-
+
+ self.multiDL = True
+ self.limitDL = 0
+
+
+ def preprocessing(self, thread):
+ """prepare"""
+ self.setup()
+ self.thread = thread
+
+ self.decrypt(self.pyfile)
+
self.createPackages()
-
+
def decrypt(self, pyfile):
raise NotImplementedError
-
- def generatePackages(self):
- """ generate new packages from self.urls """
-
- packages = map(lambda name, links: (name, links, None), self.core.api.generatePackages(self.urls).iteritems())
- self.packages.extend(packages)
-
-
def createPackages(self):
""" create new packages from self.packages """
-
- package_folder = self.pyfile.package().folder
- package_password = self.pyfile.package().password
- package_queue = self.pyfile.package().queue
-
- folder_per_package = self.config['general']['folder_per_package']
- try:
- use_subfolder = self.getConfig('use_subfolder')
- except:
- use_subfolder = folder_per_package
- try:
- subfolder_per_package = self.getConfig('subfolder_per_package')
- except:
- subfolder_per_package = True
-
for pack in self.packages:
- name, links, folder = pack
- self.logDebug("Parsed package: %s" % name,
- "%d links" % len(links),
- "Saved to folder: %s" % folder if folder else "Saved to download folder")
+ self.log.debug("Parsed package %(name)s with %(len)d links" % { "name" : pack[0], "len" : len(pack[1]) } )
+
+ links = [x.decode("utf-8") for x in pack[1]]
+
+ pid = self.core.api.addPackage(pack[0], links, self.pyfile.package().queue)
- links = map(lambda x: x.decode("utf-8"), links)
+ if self.pyfile.package().password:
+ self.core.api.setPackageData(pid, {"password": self.pyfile.package().password})
- pid = self.core.api.addPackage(name, links, package_queue)
-
- if package_password:
- self.core.api.setPackageData(pid, {"password": package_password})
-
- setFolder = lambda x: self.core.api.setPackageData(pid, {"folder": x or ""}) #: Workaround to do not break API addPackage method
-
- if use_subfolder:
- if not subfolder_per_package:
- setFolder(package_folder)
- self.logDebug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder})
-
- elif not folder_per_package or name != folder:
- if not folder:
- folder = name.replace("http://", "").replace(":", "").replace("/", "_").replace("\\", "_")
-
- folder = save_path(folder) #@TODO: move to core code
-
- setFolder(folder)
- self.logDebug("Set package %(name)s folder to: %(folder)s" % {"name": name, "folder": folder})
-
- elif folder_per_package:
- setFolder(None)
+ if self.urls:
+ self.core.api.generateAndAddPackages(self.urls)
+
diff --git a/module/plugins/Hook.py b/module/plugins/Hook.py
index 424417752..5efd08bae 100644
--- a/module/plugins/Hook.py
+++ b/module/plugins/Hook.py
@@ -1,9 +1,26 @@
# -*- coding: utf-8 -*-
-from traceback import print_exc
+"""
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License,
+ or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+ @author: mkaay
+ @interface-version: 0.2
+"""
-from module.plugins.Plugin import Base
+from traceback import print_exc
+from Plugin import Base
class Expose(object):
""" used for decoration to declare rpc services """
@@ -12,29 +29,23 @@ class Expose(object):
hookManager.addRPC(f.__module__, f.func_name, f.func_doc)
return f
-
def threaded(f):
-
def run(*args,**kwargs):
hookManager.startThread(f, *args, **kwargs)
return run
-
class Hook(Base):
"""
Base class for hook plugins.
"""
- __name__ = "Hook"
- __type__ = "hook"
- __version__ = "0.03"
-
- __config__ = [] #: [("name", "type", "desc", "default")]
-
- __description__ = """Interface for hook"""
- __license__ = "GPLv3"
- __authors__ = [("mkaay", "mkaay@mkaay.de"),
- ("RaNaN", "RaNaN@pyload.org")]
-
+ __name__ = "Hook"
+ __version__ = "0.2"
+ __type__ = "hook"
+ __threaded__ = []
+ __config__ = [ ("name", "type", "desc" , "default") ]
+ __description__ = """interface for hook"""
+ __author_name__ = ("mkaay", "RaNaN")
+ __author_mail__ = ("mkaay@mkaay.de", "RaNaN@pyload.org")
#: automatically register event listeners for functions, attribute will be deleted dont use it yourself
event_map = None
@@ -43,15 +54,15 @@ class Hook(Base):
#: List of events the plugin can handle, name the functions exactly like eventname.
event_list = None # dont make duplicate entries in event_map
+
#: periodic call interval in secondc
interval = 60
-
def __init__(self, core, manager):
Base.__init__(self, core)
#: Provide information in dict here, usable by API `getInfo`
- self.info = {}
+ self.info = None
#: Callback of periodical job task, used by hookmanager
self.cb = None
@@ -80,17 +91,15 @@ class Hook(Base):
self.initPeriodical()
self.setup()
-
def initPeriodical(self):
if self.interval >=1:
self.cb = self.core.scheduler.addJob(0, self._periodical, threaded=False)
-
def _periodical(self):
try:
if self.isActivated(): self.periodical()
except Exception, e:
- self.core.log.error(_("Error executing hooks: ") + str(e))
+ self.core.log.error(_("Error executing hooks: %s") % str(e))
if self.core.debug:
print_exc()
@@ -99,68 +108,54 @@ class Hook(Base):
def __repr__(self):
return "<Hook %s>" % self.__name__
-
-
+
def setup(self):
""" more init stuff if needed """
pass
-
def unload(self):
""" called when hook was deactivated """
pass
-
-
+
def isActivated(self):
""" checks if hook is activated"""
return self.config.getPlugin(self.__name__, "activated")
+
-
- #event methods - overwrite these if needed
+ #event methods - overwrite these if needed
def coreReady(self):
pass
-
def coreExiting(self):
pass
-
-
+
def downloadPreparing(self, pyfile):
pass
-
-
+
def downloadFinished(self, pyfile):
pass
-
-
+
def downloadFailed(self, pyfile):
pass
-
-
+
def packageFinished(self, pypack):
pass
-
def beforeReconnecting(self, ip):
pass
-
-
+
def afterReconnecting(self, ip):
pass
-
-
+
def periodical(self):
pass
-
def newCaptchaTask(self, task):
""" new captcha task for the plugin, it MUST set the handler and timeout or will be ignored """
pass
-
def captchaCorrect(self, task):
pass
-
def captchaInvalid(self, task):
- pass
+ pass \ No newline at end of file
diff --git a/module/plugins/Hoster.py b/module/plugins/Hoster.py
index be7d75bb9..814a70949 100644
--- a/module/plugins/Hoster.py
+++ b/module/plugins/Hoster.py
@@ -1,21 +1,33 @@
# -*- coding: utf-8 -*-
-from module.plugins.Plugin import Plugin
+"""
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License,
+ or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+ @author: mkaay
+"""
+
+from module.plugins.Plugin import Plugin
def getInfo(self):
#result = [ .. (name, size, status, url) .. ]
return
-
class Hoster(Plugin):
- __name__ = "Hoster"
- __type__ = "hoster"
- __version__ = "0.02"
-
+ __name__ = "Hoster"
+ __version__ = "0.1"
__pattern__ = None
- __config__ = [] #: [("name", "type", "desc", "default")]
-
+ __type__ = "hoster"
__description__ = """Base hoster plugin"""
- __license__ = "GPLv3"
- __authors__ = [("mkaay", "mkaay@mkaay.de")]
+ __author_name__ = ("mkaay")
+ __author_mail__ = ("mkaay@mkaay.de")
diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py
index 550169034..15bf3971f 100644
--- a/module/plugins/Plugin.py
+++ b/module/plugins/Plugin.py
@@ -1,5 +1,22 @@
# -*- coding: utf-8 -*-
+"""
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License,
+ or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+ @author: RaNaN, spoob, mkaay
+"""
+
from time import time, sleep
from random import randint
@@ -13,11 +30,8 @@ if os.name != "nt":
from grp import getgrnam
from itertools import islice
-from traceback import print_exc
-from urlparse import urlparse
-
-from module.utils import fs_decode, fs_encode, save_join, save_path
+from module.utils import save_join, save_path, fs_encode, fs_decode
def chunks(iterable, size):
it = iter(iterable)
@@ -60,39 +74,24 @@ class Base(object):
#: core config
self.config = core.config
-
- def _log(self, type, args):
- msg = " | ".join([str(a).strip() for a in args if a])
- logger = getattr(self.log, type)
- logger("%s: %s" % (self.__name__, msg or _("%s MARK" % type.upper())))
-
-
- def logDebug(self, *args):
- if self.core.debug:
- return self._log("debug", args)
-
-
+ #log functions
def logInfo(self, *args):
- return self._log("info", args)
-
+ self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
def logWarning(self, *args):
- return self._log("warning", args)
-
+ self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
def logError(self, *args):
- return self._log("error", args)
+ self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
-
- def logCritical(self, *args):
- return self._log("critical", args)
+ def logDebug(self, *args):
+ self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
def setConf(self, option, value):
""" see `setConfig` """
self.core.config.setPlugin(self.__name__, option, value)
-
def setConfig(self, option, value):
""" Set config value for current plugin
@@ -102,12 +101,10 @@ class Base(object):
"""
self.setConf(option, value)
-
def getConf(self, option):
""" see `getConfig` """
return self.core.config.getPlugin(self.__name__, option)
-
def getConfig(self, option):
""" Returns config value for current plugin
@@ -116,29 +113,24 @@ class Base(object):
"""
return self.getConf(option)
-
def setStorage(self, key, value):
""" Saves a value persistently to the database """
self.core.db.setStorage(self.__name__, key, value)
-
def store(self, key, value):
""" same as `setStorage` """
self.core.db.setStorage(self.__name__, key, value)
-
def getStorage(self, key=None, default=None):
""" Retrieves saved value or dict of all saved entries if key is None """
- if key:
+ if key is not None:
return self.core.db.getStorage(self.__name__, key) or default
return self.core.db.getStorage(self.__name__, key)
-
def retrieve(self, *args, **kwargs):
""" same as `getStorage` """
return self.getStorage(*args, **kwargs)
-
def delStorage(self, key):
""" Delete entry in db """
self.core.db.delStorage(self.__name__, key)
@@ -149,22 +141,14 @@ class Plugin(Base):
Base plugin for hoster/crypter.
Overwrite `process` / `decrypt` in your subclassed plugin.
"""
- __name__ = "Plugin"
- __type__ = "hoster"
- __version__ = "0.07"
-
+ __name__ = "Plugin"
+ __version__ = "0.4"
__pattern__ = None
- __config__ = [] #: [("name", "type", "desc", "default")]
-
- __description__ = """Base plugin"""
- __license__ = "GPLv3"
- __authors__ = [("RaNaN", "RaNaN@pyload.org"),
- ("spoob", "spoob@pyload.org"),
- ("mkaay", "mkaay@mkaay.de")]
-
-
- info = {} #: file info dict
-
+ __type__ = "hoster"
+ __config__ = [("name", "type", "desc", "default")]
+ __description__ = """Base Plugin"""
+ __author_name__ = ("RaNaN", "spoob", "mkaay")
+ __author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de")
def __init__(self, pyfile):
Base.__init__(self, pyfile.m.core)
@@ -190,9 +174,7 @@ class Plugin(Base):
#: username/login
self.user = None
- if self.account and not self.account.canUse():
- self.account = None
-
+ if self.account and not self.account.canUse(): self.account = None
if self.account:
self.user, data = self.account.selectAccount()
#: Browser instance, see `network.Browser`
@@ -218,32 +200,27 @@ class Plugin(Base):
self.js = self.core.js
self.cTask = None #captcha task
- self.html = None #@TODO: Move to hoster class
self.retries = 0 # amount of retries already made
+ self.html = None # some plugins store html code here
self.init()
-
def getChunkCount(self):
if self.chunkLimit <= 0:
- return self.config['download']['chunks']
- return min(self.config['download']['chunks'], self.chunkLimit)
-
+ return self.config["download"]["chunks"]
+ return min(self.config["download"]["chunks"], self.chunkLimit)
def __call__(self):
return self.__name__
-
def init(self):
"""initialize the plugin (in addition to `__init__`)"""
pass
-
def setup(self):
""" setup for enviroment and other things, called before downloading (possibly more than one time)"""
pass
-
def preprocessing(self, thread):
""" handles important things to do before starting """
self.thread = thread
@@ -264,14 +241,12 @@ class Plugin(Base):
"""the 'main' method of every plugin, you **have to** overwrite it"""
raise NotImplementedError
-
def resetAccount(self):
""" dont use account and retry download """
self.account = None
self.req = self.core.requestFactory.getRequest(self.__name__)
self.retry()
-
def checksum(self, local_file=None):
"""
return codes:
@@ -286,114 +261,46 @@ class Plugin(Base):
return True, 10
- def setReconnect(self, reconnect):
- reconnect = bool(reconnect)
- self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect))
- self.wantReconnect = reconnect
-
-
- def setWait(self, seconds, reconnect=None):
+ def setWait(self, seconds, reconnect=False):
"""Set a specific wait time later used with `wait`
-
+
:param seconds: wait time in seconds
:param reconnect: True if a reconnect would avoid wait time
"""
- wait_time = int(seconds) + 1
- wait_until = time() + wait_time
-
- self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil),
- "Wait: %d seconds" % wait_time)
+ if reconnect:
+ self.wantReconnect = True
+ self.pyfile.waitUntil = time() + int(seconds)
- self.pyfile.waitUntil = wait_until
-
- if reconnect is not None:
- self.setReconnect(reconnect)
-
-
- def wait(self, seconds=None, reconnect=None):
+ def wait(self):
""" waits the time previously set """
-
- pyfile = self.pyfile
-
- if seconds is not None:
- self.setWait(seconds)
-
- if reconnect is not None:
- self.setReconnect(reconnect)
-
self.waiting = True
+ self.pyfile.setStatus("waiting")
- status = pyfile.status
- pyfile.setStatus("waiting")
+ while self.pyfile.waitUntil > time():
+ self.thread.m.reconnecting.wait(2)
- self.logInfo(_("Wait: %d seconds") % (pyfile.waitUntil - time()),
- _("Reconnect: %s") % self.wantReconnect)
-
- if self.account:
- self.logDebug("Ignore reconnection due account logged")
-
- while pyfile.waitUntil > time():
- if pyfile.abort:
- self.abort()
-
- sleep(1)
- else:
- while pyfile.waitUntil > time():
- self.thread.m.reconnecting.wait(2)
-
- if pyfile.abort:
- self.abort()
-
- if self.thread.m.reconnecting.isSet():
- self.waiting = False
- self.wantReconnect = False
- raise Reconnect
-
- sleep(1)
+ if self.pyfile.abort: raise Abort
+ if self.thread.m.reconnecting.isSet():
+ self.waiting = False
+ self.wantReconnect = False
+ raise Reconnect
self.waiting = False
-
- pyfile.status = status
-
+ self.pyfile.setStatus("starting")
def fail(self, reason):
""" fail and give reason """
raise Fail(reason)
-
- def abort(self, reason=""):
- """ abort and give reason """
- if reason:
- self.pyfile.error = str(reason)
- raise Abort #@TODO: Use raise Abort(reason) in 0.4.10
-
-
- def error(self, reason="", type=""):
- if not reason and not type:
- type = "unknown"
-
- msg = _("%s error") % _(type.strip().capitalize()) if type else _("Error")
- msg += ": " + reason.strip() if reason else ""
- msg += _(" | Plugin may be out of date")
-
- raise Fail(msg)
-
-
- def offline(self, reason=""):
+ def offline(self):
""" fail and indicate file is offline """
- if reason:
- self.pyfile.error = str(reason)
- raise Fail("offline") #@TODO: Use raise Fail("offline", reason) in 0.4.10
+ raise Fail("offline")
-
- def tempOffline(self, reason=""):
+ def tempOffline(self):
""" fail and indicates file ist temporary offline, the core may take consequences """
- if reason:
- self.pyfile.error = str(reason)
- raise Fail("temp. offline") #@TODO: Use raise Fail("temp. offline", reason) in 0.4.10
-
+ raise Fail("temp. offline")
- def retry(self, max_tries=5, wait_time=1, reason=""):
+ def retry(self, max_tries=3, wait_time=1, reason=""):
"""Retries and begin again from the beginning
:param max_tries: number of maximum retries
@@ -401,28 +308,26 @@ class Plugin(Base):
:param reason: reason for retrying, will be passed to fail if max_tries reached
"""
if 0 < max_tries <= self.retries:
- self.error(reason or _("Max retries reached"), "retry")
+ if not reason: reason = "Max retries reached"
+ raise Fail(reason)
- self.wait(wait_time, False)
+ self.wantReconnect = False
+ self.setWait(wait_time)
+ self.wait()
self.retries += 1
raise Retry(reason)
-
def invalidCaptcha(self):
- self.logError(_("Invalid captcha"))
if self.cTask:
self.cTask.invalid()
-
def correctCaptcha(self):
- self.logInfo(_("Correct captcha"))
if self.cTask:
self.cTask.correct()
-
def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg',
- result_type='textual', timeout=290):
+ result_type='textual'):
""" Loads a captcha and decrypts it with ocr, plugin, user input
:param url: url of captcha image
@@ -434,16 +339,16 @@ class Plugin(Base):
:param result_type: 'textual' if text is written on the captcha\
or 'positional' for captcha where the user have to click\
on a specific region on the captcha
-
+
:return: result of decrypting
"""
img = self.load(url, get=get, post=post, cookies=cookies)
id = ("%.2f" % time())[-6:].replace(".", "")
-
- with open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha:
- tmpCaptcha.write(img)
+ temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb")
+ temp_file.write(img)
+ temp_file.close()
has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins
@@ -454,21 +359,20 @@ class Plugin(Base):
if Ocr and not forceUser:
sleep(randint(3000, 5000) / 1000.0)
- if self.pyfile.abort:
- self.abort()
+ if self.pyfile.abort: raise Abort
ocr = Ocr()
- result = ocr.get_captcha(tmpCaptcha.name)
+ result = ocr.get_captcha(temp_file.name)
else:
captchaManager = self.core.captchaManager
- task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type)
+ task = captchaManager.newTask(img, imgtype, temp_file.name, result_type)
self.cTask = task
- captchaManager.handleCaptcha(task, timeout)
+ captchaManager.handleCaptcha(task)
while task.isWaiting():
if self.pyfile.abort:
captchaManager.removeTask(task)
- self.abort()
+ raise Abort
sleep(1)
captchaManager.removeTask(task)
@@ -478,21 +382,21 @@ class Plugin(Base):
elif task.error:
self.fail(task.error)
elif not task.result:
- self.fail(_("No captcha result obtained in appropiate time by any of the plugins"))
+ self.fail(_("No captcha result obtained in appropiate time by any of the plugins."))
result = task.result
- self.logDebug("Received captcha result: " + str(result))
+ self.log.debug("Received captcha result: %s" % str(result))
if not self.core.debug:
try:
- remove(tmpCaptcha.name)
+ remove(temp_file.name)
except:
pass
return result
- def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True):
+ def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False):
"""Load content at url and returns it
:param url:
@@ -500,44 +404,35 @@ class Plugin(Base):
:param post:
:param ref:
:param cookies:
- :param just_header: If True only the header will be retrieved and returned as dict
+ :param just_header: if True only the header will be retrieved and returned as dict
:param decode: Wether to decode the output according to http header, should be True in most cases
- :param follow_location: If True follow location else not
- :param save_cookies: If True saves received cookies else discard them
:return: Loaded content
"""
- if self.pyfile.abort:
- self.abort()
-
- if not url:
- self.fail(_("No url given"))
-
- if type(url) == unicode: # utf8 vs decode -> please use decode attribute in all future plugins
- url = str(url) #: encode('utf8')
-
- url = url.strip()
-
- if self.core.debug:
- self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])
+ if self.pyfile.abort: raise Abort
+ #utf8 vs decode -> please use decode attribute in all future plugins
+ if type(url) == unicode: url = str(url)
- res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies)
+ res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode)
if self.core.debug:
from inspect import currentframe
frame = currentframe()
- framefile = save_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
+ if not exists(join("tmp", self.__name__)):
+ makedirs(join("tmp", self.__name__))
+
+ f = open(
+ join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
+ , "wb")
+ del frame # delete the frame or it wont be cleaned
+
try:
- if not exists(join("tmp", self.__name__)):
- makedirs(join("tmp", self.__name__))
+ tmp = res.encode("utf8")
+ except:
+ tmp = res
- with open(framefile, "wb") as f:
- del frame #: delete the frame or it wont be cleaned
- if decode:
- res = res.encode('utf-8')
- f.write(res)
- except IOError, e:
- self.logError(e)
+ f.write(tmp)
+ f.close()
if just_header:
#parse header
@@ -547,7 +442,7 @@ class Plugin(Base):
if not line or ":" not in line: continue
key, none, value = line.partition(":")
- key = key.strip().lower()
+ key = key.lower().strip()
value = value.strip()
if key in header:
@@ -561,7 +456,6 @@ class Plugin(Base):
return res
-
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
"""Downloads the content at url to download folder
@@ -574,19 +468,6 @@ class Plugin(Base):
the filename will be changed if needed
:return: The location where the file was saved
"""
- if self.pyfile.abort:
- self.abort()
-
- if not url:
- self.fail(_("No url given"))
-
- if type(url) == unicode:
- url = str(url)
-
- url = url.strip()
-
- if self.core.debug:
- self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])
self.checkForSameFiles()
@@ -597,16 +478,16 @@ class Plugin(Base):
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
- try:
- makedirs(location, int(self.core.config['permission']['folder'], 8))
+ makedirs(location, int(self.core.config["permission"]["folder"], 8))
- if self.core.config['permission']['change_dl'] and os.name != "nt":
- uid = getpwnam(self.config['permission']['user'])[2]
- gid = getgrnam(self.config['permission']['group'])[2]
- chown(location, uid, gid)
+ if self.core.config["permission"]["change_dl"] and os.name != "nt":
+ try:
+ uid = getpwnam(self.config["permission"]["user"])[2]
+ gid = getgrnam(self.config["permission"]["group"])[2]
- except Exception, e:
- self.fail(e)
+ chown(location, uid, gid)
+ except Exception, e:
+ self.log.warning(_("Setting User and Group failed: %s") % str(e))
# convert back to unicode
location = fs_decode(location)
@@ -623,38 +504,31 @@ class Plugin(Base):
finally:
self.pyfile.size = self.req.size
- if newname:
- newname = urlparse(newname).path.split("/")[-1]
-
- if disposition and newname != name:
- self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname})
- self.pyfile.name = newname
- filename = join(location, newname)
+ if disposition and newname and newname != name: #triple check, just to be sure
+ self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname})
+ self.pyfile.name = newname
+ filename = join(location, newname)
fs_filename = fs_encode(filename)
- if self.core.config['permission']['change_file']:
- try:
- chmod(fs_filename, int(self.core.config['permission']['file'], 8))
- except Exception, e:
- self.logWarning(_("Setting file mode failed"), e)
+ if self.core.config["permission"]["change_file"]:
+ chmod(fs_filename, int(self.core.config["permission"]["file"], 8))
- if self.core.config['permission']['change_dl'] and os.name != "nt":
+ if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
- uid = getpwnam(self.config['permission']['user'])[2]
- gid = getgrnam(self.config['permission']['group'])[2]
- chown(fs_filename, uid, gid)
+ uid = getpwnam(self.config["permission"]["user"])[2]
+ gid = getgrnam(self.config["permission"]["group"])[2]
+ chown(fs_filename, uid, gid)
except Exception, e:
- self.logWarning(_("Setting User and Group failed"), e)
+ self.log.warning(_("Setting User and Group failed: %s") % str(e))
self.lastDownload = filename
return self.lastDownload
-
def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0):
""" checks the content of the last downloaded file, re match is saved to `lastCheck`
-
+
:param rules: dict with names and rules to match (compiled regexp or strings)
:param api_size: expected file size
:param max_size: if the file is larger then it wont be checked
@@ -663,21 +537,19 @@ class Plugin(Base):
:return: dictionary key of the first rule that matched
"""
lastDownload = fs_encode(self.lastDownload)
- if not exists(lastDownload):
- return None
+ if not exists(lastDownload): return None
size = stat(lastDownload)
size = size.st_size
if api_size and api_size <= size: return None
elif size > max_size and not read_size: return None
- self.logDebug("Download Check triggered")
-
- with open(lastDownload, "rb") as f:
- content = f.read(read_size if read_size else -1)
-
+ self.log.debug("Download Check triggered")
+ f = open(lastDownload, "rb")
+ content = f.read(read_size if read_size else -1)
+ f.close()
#produces encoding errors, better log to other file in the future?
- #self.logDebug("Content: %s" % content)
+ #self.log.debug("Content: %s" % content)
for name, rule in rules.iteritems():
if type(rule) in (str, unicode):
if rule in content:
@@ -723,27 +595,23 @@ class Plugin(Base):
if starting and self.core.config['download']['skip_existing'] and exists(location):
size = os.stat(location).st_size
if size >= self.pyfile.size:
- raise SkipDownload("File exists")
+ raise SkipDownload("File exists.")
pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
if pyfile:
if exists(location):
raise SkipDownload(pyfile[0])
- self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name)
-
+ self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
def clean(self):
""" clean everything and remove references """
if hasattr(self, "pyfile"):
del self.pyfile
-
if hasattr(self, "req"):
self.req.close()
del self.req
-
if hasattr(self, "thread"):
del self.thread
-
if hasattr(self, "html"):
del self.html
diff --git a/module/plugins/PluginManager.py b/module/plugins/PluginManager.py
index 59ba47410..f3f5f47bc 100644
--- a/module/plugins/PluginManager.py
+++ b/module/plugins/PluginManager.py
@@ -1,26 +1,43 @@
# -*- coding: utf-8 -*-
+"""
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 3 of the License,
+ or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+ @author: mkaay, RaNaN
+"""
+
import re
import sys
-from itertools import chain
from os import listdir, makedirs
from os.path import isfile, join, exists, abspath
from sys import version_info
+from itertools import chain
from traceback import print_exc
from module.lib.SafeEval import const_eval as literal_eval
-
+from module.ConfigParser import IGNORE
class PluginManager:
- ROOT = "module.plugins."
+ ROOT = "module.plugins."
USERROOT = "userplugins."
- TYPES = ("crypter", "container", "hoster", "captcha", "accounts", "hooks", "internal")
+ TYPES = ("crypter", "container", "hoster", "captcha", "accounts", "hooks", "internal")
- PATTERN = re.compile(r'__pattern__\s*=\s*u?r("|\')([^"\']+)')
- VERSION = re.compile(r'__version__\s*=\s*("|\')([\d.]+)')
- CONFIG = re.compile(r'__config__\s*=\s*\[([^\]]+)', re.M)
- DESC = re.compile(r'__description__\s*=\s*("|"""|\')([^"\']+)')
+ PATTERN = re.compile(r'__pattern__.*=.*r("|\')([^"\']+)')
+ VERSION = re.compile(r'__version__.*=.*("|\')([0-9.]+)')
+ CONFIG = re.compile(r'__config__.*=.*\[([^\]]+)', re.MULTILINE)
+ DESC = re.compile(r'__description__.?=.?("|"""|\')([^"\']+)')
def __init__(self, core):
@@ -41,40 +58,41 @@ class PluginManager:
sys.path.append(abspath(""))
- #@NOTE: In 0.4.10 directory "accounts" changes to "account" and "hooks" changes to "addon"
- self.plugins['accounts'] = self.accountPlugins = self.parse("accounts")
- self.plugins['hooks'] = self.hookPlugins = self.parse("hooks")
+ if not exists("userplugins"):
+ makedirs("userplugins")
+ if not exists(join("userplugins", "__init__.py")):
+ f = open(join("userplugins", "__init__.py"), "wb")
+ f.close()
- for type in set(self.TYPES) - set(('accounts', 'hooks')):
- self.plugins[type] = self.parse(type)
- setattr(self, "%sPlugins" % type, self.plugins[type])
+ self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True)
+ self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True)
+ self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True)
- self.log.debug("Created index of plugins")
+ self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha")
+ self.plugins["accounts"] = self.accountPlugins = self.parse("accounts")
+ self.plugins["hooks"] = self.hookPlugins = self.parse("hooks")
+ self.plugins["internal"] = self.internalPlugins = self.parse("internal")
+ self.log.debug("created index of plugins")
- def parse(self, folder, rootplugins={}):
+ def parse(self, folder, pattern=False, home={}):
"""
- returns dict with information
+ returns dict with information
home contains parsed plugins from module.
+
+ {
+ name : {path, version, config, (pattern, re), (plugin, class)}
+ }
+
"""
-
plugins = {}
-
- if rootplugins:
- try:
- pfolder = join("userplugins", folder)
- if not exists(pfolder):
- makedirs(pfolder)
-
- for ifile in (join("userplugins", "__init__.py"),
- join(pfolder, "__init__.py")):
- if not exists(ifile):
- f = open(ifile, "wb")
- f.close()
-
- except IOError, e:
- self.logCritical(e)
- return rootplugins
+ if home:
+ pfolder = join("userplugins", folder)
+ if not exists(pfolder):
+ makedirs(pfolder)
+ if not exists(join(pfolder, "__init__.py")):
+ f = open(join(pfolder, "__init__.py"), "wb")
+ f.close()
else:
pfolder = join(pypath, "module", "plugins", folder)
@@ -82,27 +100,19 @@ class PluginManager:
for f in listdir(pfolder):
if (isfile(join(pfolder, f)) and f.endswith(".py") or f.endswith("_25.pyc") or f.endswith(
"_26.pyc") or f.endswith("_27.pyc")) and not f.startswith("_"):
+ data = open(join(pfolder, f))
+ content = data.read()
+ data.close()
- try:
- with open(join(pfolder, f)) as data:
- content = data.read()
-
- except IOError, e:
- self.logError(e)
+ if f.endswith("_25.pyc") and version_info[0:2] != (2, 5):
continue
-
- if f.endswith("_25.pyc") and version_info[0:2] != (2, 5): #@TODO: Remove in 0.4.10
+ elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6):
continue
-
- elif f.endswith("_26.pyc") and version_info[0:2] != (2, 6): #@TODO: Remove in 0.4.10
- continue
-
- elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7): #@TODO: Remove in 0.4.10
+ elif f.endswith("_27.pyc") and version_info[0:2] != (2, 7):
continue
name = f[:-3]
- if name[-1] == ".":
- name = name[:-4]
+ if name[-1] == ".": name = name[:-4]
version = self.VERSION.findall(content)
if version:
@@ -110,33 +120,38 @@ class PluginManager:
else:
version = 0
- if rootplugins and name in rootplugins:
- if rootplugins[name]['version'] >= version:
+ # home contains plugins from pyload root
+ if home and name in home:
+ if home[name]["v"] >= version:
continue
+ if name in IGNORE or (folder, name) in IGNORE:
+ continue
+
plugins[name] = {}
- plugins[name]['version'] = version
+ plugins[name]["v"] = version
module = f.replace(".pyc", "").replace(".py", "")
# the plugin is loaded from user directory
- plugins[name]['user'] = True if rootplugins else False
- plugins[name]['name'] = module
-
- pattern = self.PATTERN.findall(content)
+ plugins[name]["user"] = True if home else False
+ plugins[name]["name"] = module
if pattern:
- pattern = pattern[0][1]
+ pattern = self.PATTERN.findall(content)
+
+ if pattern:
+ pattern = pattern[0][1]
+ else:
+ pattern = "^unmachtable$"
+
+ plugins[name]["pattern"] = pattern
try:
- regexp = re.compile(pattern)
+ plugins[name]["re"] = re.compile(pattern)
except:
- self.log.error(_("%s has a invalid pattern") % name)
- pattern = r'^unmatchable$'
- regexp = re.compile(pattern)
+ self.log.error(_("%s has a invalid pattern.") % name)
- plugins[name]['pattern'] = pattern
- plugins[name]['re'] = regexp
# internals have no config
if folder == "internal":
@@ -145,19 +160,24 @@ class PluginManager:
config = self.CONFIG.findall(content)
if config:
- try:
- config = literal_eval(config[0].strip().replace("\n", "").replace("\r", ""))
- desc = self.DESC.findall(content)
- desc = desc[0][1] if desc else ""
+ config = literal_eval(config[0].strip().replace("\n", "").replace("\r", ""))
+ desc = self.DESC.findall(content)
+ desc = desc[0][1] if desc else ""
- if type(config[0]) == tuple:
- config = [list(x) for x in config]
- else:
- config = [list(config)]
+ if type(config[0]) == tuple:
+ config = [list(x) for x in config]
+ else:
+ config = [list(config)]
- if folder not in ("accounts", "internal") and not [True for item in config if item[0] == "activated"]:
- config.insert(0, ["activated", "bool", "Activated", False if folder == "hooks" else True])
+ if folder == "hooks":
+ append = True
+ for item in config:
+ if item[0] == "activated": append = False
+ # activated flag missing
+ if append: config.append(["activated", "bool", "Activated", False])
+
+ try:
self.core.config.addPluginConfig(name, config, desc)
except:
self.log.error("Invalid config in %s: %s" % (name, config))
@@ -172,8 +192,9 @@ class PluginManager:
except:
self.log.error("Invalid config in %s: %s" % (name, config))
- if not rootplugins and plugins: #: Double check
- plugins.update(self.parse(folder, plugins))
+ if not home:
+ temp = self.parse(folder, pattern, plugins)
+ plugins.update(temp)
return plugins
@@ -188,13 +209,13 @@ class PluginManager:
if type(url) not in (str, unicode, buffer): continue
found = False
- if last and last[1]['re'].match(url):
+ if last and last[1]["re"].match(url):
res.append((url, last[0]))
continue
for name, value in chain(self.crypterPlugins.iteritems(), self.hosterPlugins.iteritems(),
- self.containerPlugins.iteritems()):
- if 're' in value and value['re'].match(url): #@TODO: Rewrite this check to report missing __pattern__ attribute alert
+ self.containerPlugins.iteritems()):
+ if value["re"].match(url):
res.append((url, name))
last = (name, value)
found = True
@@ -205,38 +226,34 @@ class PluginManager:
return res
-
def findPlugin(self, name, pluginlist=("hoster", "crypter", "container")):
for ptype in pluginlist:
if name in self.plugins[ptype]:
return self.plugins[ptype][name], ptype
return None, None
-
def getPlugin(self, name, original=False):
"""return plugin module from hoster|decrypter|container"""
plugin, type = self.findPlugin(name)
if not plugin:
- self.log.warning("Plugin %s not found" % name)
- plugin = self.hosterPlugins['BasePlugin']
+ self.log.warning("Plugin %s not found." % name)
+ plugin = self.hosterPlugins["BasePlugin"]
if "new_module" in plugin and not original:
- return plugin['new_module']
+ return plugin["new_module"]
return self.loadModule(type, name)
-
def getPluginName(self, name):
""" used to obtain new name if other plugin was injected"""
plugin, type = self.findPlugin(name)
if "new_name" in plugin:
- return plugin['new_name']
+ return plugin["new_name"]
return name
-
def loadModule(self, type, name):
""" Returns loaded module for plugin
@@ -245,41 +262,26 @@ class PluginManager:
"""
plugins = self.plugins[type]
if name in plugins:
- if "module" in plugins[name]:
- return plugins[name]['module']
-
+ if "module" in plugins[name]: return plugins[name]["module"]
try:
- module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]['name']), globals(), locals(),
- plugins[name]['name'])
-
+ module = __import__(self.ROOT + "%s.%s" % (type, plugins[name]["name"]), globals(), locals(),
+ plugins[name]["name"])
+ plugins[name]["module"] = module #cache import, maybe unneeded
+ return module
except Exception, e:
- self.log.error(_("Error importing plugin: [%(type)s] %(name)s (v%(version).2f) | %(errmsg)s")
- % {'name': name, 'type': type, 'version': plugins[name]['version'], "errmsg": str(e)})
+ self.log.error(_("Error importing %(name)s: %(msg)s") % {"name": name, "msg": str(e)})
if self.core.debug:
print_exc()
- else:
- plugins[name]['module'] = module #: cache import, maybe unneeded
-
- self.log.debug(_("Loaded plugin: [%(type)s] %(name)s (v%(version).2f)")
- % {'name': name, 'type': type, 'version': plugins[name]['version']})
- return module
-
-
def loadClass(self, type, name):
"""Returns the class of a plugin with the same name"""
module = self.loadModule(type, name)
- if module:
- return getattr(module, name)
- else:
- return None
-
+ if module: return getattr(module, name)
def getAccountPlugins(self):
"""return list of account plugin names"""
return self.accountPlugins.keys()
-
def find_module(self, fullname, path=None):
#redirecting imports if necesarry
if fullname.startswith(self.ROOT) or fullname.startswith(self.USERROOT): #seperate pyload plugins
@@ -292,10 +294,10 @@ class PluginManager:
if type in self.plugins and name in self.plugins[type]:
#userplugin is a newer version
- if not user and self.plugins[type][name]['user']:
+ if not user and self.plugins[type][name]["user"]:
return self
#imported from userdir, but pyloads is newer
- if user and not self.plugins[type][name]['user']:
+ if user and not self.plugins[type][name]["user"]:
return self
@@ -306,8 +308,7 @@ class PluginManager:
newname = name.replace(self.ROOT, self.USERROOT)
else:
newname = name.replace(self.USERROOT, self.ROOT)
- else:
- newname = name
+ else: newname = name
base, plugin = newname.rsplit(".", 1)
@@ -323,45 +324,40 @@ class PluginManager:
def reloadPlugins(self, type_plugins):
""" reloads and reindexes plugins """
- if not type_plugins:
- return None
+ if not type_plugins: return False
self.log.debug("Request reload of plugins: %s" % type_plugins)
- flag = True
as_dict = {}
-
for t,n in type_plugins:
if t in as_dict:
as_dict[t].append(n)
else:
as_dict[t] = [n]
- for type in as_dict.iterkeys():
- # we do not reload hooks or internals, would cause to much side effects
- if type in ("hooks", "internal"):
- flag = False
- continue
+ # we do not reload hooks or internals, would cause to much side effects
+ if "hooks" in as_dict or "internal" in as_dict:
+ return False
+ for type in as_dict.iterkeys():
for plugin in as_dict[type]:
if plugin in self.plugins[type]:
if "module" in self.plugins[type][plugin]:
- self.log.debug("Reloading module %s" % plugin)
- reload(self.plugins[type][plugin]['module'])
+ self.log.debug("Reloading %s" % plugin)
+ reload(self.plugins[type][plugin]["module"])
- #index creation
- self.plugins[type] = self.parse(type)
-
- if type is "accounts": #@TODO: Remove this check in 0.4.10
- self.accountPlugins = self.plugins[type]
- else:
- setattr(self, "%sPlugins" % type, self.plugins[type])
+ #index creation
+ self.plugins["crypter"] = self.crypterPlugins = self.parse("crypter", pattern=True)
+ self.plugins["container"] = self.containerPlugins = self.parse("container", pattern=True)
+ self.plugins["hoster"] = self.hosterPlugins = self.parse("hoster", pattern=True)
+ self.plugins["captcha"] = self.captchaPlugins = self.parse("captcha")
+ self.plugins["accounts"] = self.accountPlugins = self.parse("accounts")
- if "accounts" in as_dict: #: accounts needs to be reloaded
+ if "accounts" in as_dict: #accounts needs to be reloaded
self.core.accountManager.initPlugins()
self.core.scheduler.addJob(0, self.core.accountManager.getAccountInfos)
- return flag
+ return True
@@ -375,9 +371,10 @@ if __name__ == "__main__":
a = time()
- test = ["http://www.youtube.com/watch?v=%s" % x for x in xrange(0, 100)]
+ test = ["http://www.youtube.com/watch?v=%s" % x for x in range(0, 100)]
print p.parseUrls(test)
b = time()
print b - a, "s"
+