summaryrefslogtreecommitdiffstats
path: root/pyload/plugins/Plugin.py
diff options
context:
space:
mode:
Diffstat (limited to 'pyload/plugins/Plugin.py')
-rw-r--r--pyload/plugins/Plugin.py329
1 files changed, 225 insertions, 104 deletions
diff --git a/pyload/plugins/Plugin.py b/pyload/plugins/Plugin.py
index 98153bd5b..43dbcfab3 100644
--- a/pyload/plugins/Plugin.py
+++ b/pyload/plugins/Plugin.py
@@ -13,8 +13,10 @@ if os.name != "nt":
from grp import getgrnam
from itertools import islice
+from traceback import print_exc
+from urlparse import urlparse
-from pyload.utils import safe_join, safe_filename, fs_encode, fs_decode
+from module.utils import fs_decode, fs_encode, html_unescape, save_join, save_path
def chunks(iterable, size):
it = iter(iterable)
@@ -57,23 +59,39 @@ class Base(object):
#: core config
self.config = core.config
- #log functions
+
+ def _log(self, type, args):
+ msg = " | ".join([str(a).strip() for a in args if a])
+ logger = getattr(self.log, type)
+ logger("%s: %s" % (self.__name__, msg or _("%s MARK" % type.upper())))
+
+
+ def logDebug(self, *args):
+ if self.core.debug:
+ return self._log("debug", args)
+
+
def logInfo(self, *args):
- self.log.info("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
+ return self._log("info", args)
+
def logWarning(self, *args):
- self.log.warning("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
+ return self._log("warning", args)
+
def logError(self, *args):
- self.log.error("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
+ return self._log("error", args)
- def logDebug(self, *args):
- self.log.debug("%s: %s" % (self.__name__, " | ".join([a if isinstance(a, basestring) else str(a) for a in args])))
+
+ def logCritical(self, *args):
+ return self._log("critical", args)
+ #: Deprecated method
def setConf(self, option, value):
""" see `setConfig` """
- self.config.setPlugin(self.__name__, option, value)
+ self.setConfig(option, value)
+
def setConfig(self, option, value):
""" Set config value for current plugin
@@ -82,39 +100,46 @@ class Base(object):
:param value:
:return:
"""
- self.setConf(option, value)
+ self.core.config.setPlugin(self.__name__, option, value)
+
#: Deprecated method
def getConf(self, option):
""" see `getConfig` """
return self.getConfig(option)
+
def getConfig(self, option):
""" Returns config value for current plugin
:param option:
:return:
"""
- return self.config.getPlugin(self.__name__, option)
+ return self.core.config.getPlugin(self.__name__, option)
+
def setStorage(self, key, value):
""" Saves a value persistently to the database """
self.core.db.setStorage(self.__name__, key, value)
+
def store(self, key, value):
""" same as `setStorage` """
self.core.db.setStorage(self.__name__, key, value)
+
def getStorage(self, key=None, default=None):
""" Retrieves saved value or dict of all saved entries if key is None """
- if key is not None:
+ if key:
return self.core.db.getStorage(self.__name__, key) or default
return self.core.db.getStorage(self.__name__, key)
+
def retrieve(self, *args, **kwargs):
""" same as `getStorage` """
return self.getStorage(*args, **kwargs)
+
def delStorage(self, key):
""" Delete entry in db """
self.core.db.delStorage(self.__name__, key)
@@ -125,17 +150,21 @@ class Plugin(Base):
Base plugin for hoster/crypter.
Overwrite `process` / `decrypt` in your subclassed plugin.
"""
- __name__ = "Plugin"
- __type__ = "hoster"
- __version__ = "0.5"
+ __name__ = "Plugin"
+ __type__ = "hoster"
+ __version__ = "0.07"
__pattern__ = None
- __config__ = [("name", "type", "desc", "default")]
+ __config__ = [] #: [("name", "type", "desc", "default")]
__description__ = """Base plugin"""
- __authors__ = [("RaNaN", "RaNaN@pyload.org"),
- ("spoob", "spoob@pyload.org"),
- ("mkaay", "mkaay@mkaay.de")]
+ __license__ = "GPLv3"
+ __authors__ = [("RaNaN", "RaNaN@pyload.org"),
+ ("spoob", "spoob@pyload.org"),
+ ("mkaay", "mkaay@mkaay.de")]
+
+
+ info = {} #: file info dict
def __init__(self, pyfile):
@@ -199,33 +228,37 @@ class Plugin(Base):
#: captcha task
self.cTask = None
- #: amount of retries already made
+ self.html = None #@TODO: Move to hoster class
self.retries = 0
#: some plugins store html code here
- self.html = None
#: quick caller for API
- self.api = self.core.api
+ # self.api = self.core.api
self.init()
+
def getChunkCount(self):
if self.chunkLimit <= 0:
return self.config['download']['chunks']
return min(self.config['download']['chunks'], self.chunkLimit)
+
def __call__(self):
return self.__name__
+
def init(self):
"""initialize the plugin (in addition to `__init__`)"""
pass
+
def setup(self):
""" setup for enviroment and other things, called before downloading (possibly more than one time)"""
pass
+
def preprocessing(self, thread):
""" handles important things to do before starting """
self.thread = thread
@@ -246,12 +279,14 @@ class Plugin(Base):
"""the 'main' method of every plugin, you **have to** overwrite it"""
raise NotImplementedError
+
def resetAccount(self):
""" dont use account and retry download """
self.account = None
self.req = self.core.requestFactory.getRequest(self.__name__)
self.retry()
+
def checksum(self, local_file=None):
"""
return codes:
@@ -272,48 +307,96 @@ class Plugin(Base):
:param seconds: wait time in seconds
:param reconnect: True if a reconnect would avoid wait time
"""
- if reconnect:
- self.wantReconnect = True
- self.pyfile.waitUntil = time() + int(seconds)
+ wait_time = int(seconds) + 1
+ wait_until = time() + wait_time
- def wait(self, seconds=None, reconnect=None):
- """ Waits the time previously set or use these from arguments. See `setWait`
- """
- if seconds:
- self.setWait(seconds, reconnect)
+ self.logDebug("Set waitUntil to: %f (previous: %f)" % (wait_until, self.pyfile.waitUntil),
+ "Wait: %d seconds" % wait_time)
+
+ self.pyfile.waitUntil = wait_until
+
+ if reconnect is not None:
+ self.logDebug("Set wantReconnect to: %s (previous: %s)" % (reconnect, self.wantReconnect))
+ self.wantReconnect = reconnect
+
+
+ def wait(self, seconds=0, reconnect=None):
+ """ waits the time previously set """
+
+ pyfile = self.pyfile
- self._wait()
+ self.setWait(seconds, reconnect)
- def _wait(self):
self.waiting = True
- self.pyfile.setStatus("waiting")
- while self.pyfile.waitUntil > time():
- self.thread.m.reconnecting.wait(2)
+ status = pyfile.status
+ pyfile.setStatus("waiting")
- if self.pyfile.abort:
- raise Abort
- if self.thread.m.reconnecting.isSet():
- self.waiting = False
- self.wantReconnect = False
- raise Reconnect
+ self.logDebug("WAIT: %d seconds" % seconds,
+ "WAITUNTIL: %f" % pyfile.waitUntil,
+ "RECONNECT: %s" % self.wantReconnect)
+
+ if not account:
+ self.logDebug("Ignore reconnection due account logged")
+
+ while pyfile.waitUntil > time():
+ self.thread.m.reconnecting.wait(2)
+
+ if pyfile.abort:
+ self.abort()
+
+ if self.thread.m.reconnecting.isSet():
+ self.waiting = False
+ self.wantReconnect = False
+ raise Reconnect
+ else:
+ while pyfile.waitUntil > time():
+ if pyfile.abort:
+ self.abort()
self.waiting = False
- self.pyfile.setStatus("starting")
+
+ pyfile.status = status
+
def fail(self, reason):
""" fail and give reason """
raise Fail(reason)
- def offline(self):
+
+ def abort(self, reason=""):
+ """ abort and give reason """
+ if reason:
+ self.pyfile.error = str(reason)
+ raise Abort #@TODO: Use raise Abort(reason) in 0.4.10
+
+
+ def error(self, reason="", type=""):
+ if not reason and not type:
+ type = "unknown"
+
+ msg = _("%s error") % _(type.strip().capitalize()) if type else _("Error")
+ msg += ": " + reason.strip() if reason else ""
+ msg += _(" | Plugin may be out of date")
+
+ raise Fail(msg)
+
+
+ def offline(self, reason=""):
""" fail and indicate file is offline """
- raise Fail("offline")
+ if reason:
+ self.pyfile.error = str(reason)
+ raise Fail("offline") #@TODO: Use raise Fail("offline", reason) in 0.4.10
+
- def tempOffline(self):
+ def tempOffline(self, reason=""):
""" fail and indicates file ist temporary offline, the core may take consequences """
- raise Fail("temp. offline")
+ if reason:
+ self.pyfile.error = str(reason)
+ raise Fail("temp. offline") #@TODO: Use raise Fail("temp. offline", reason) in 0.4.10
- def retry(self, max_tries=3, wait_time=1, reason=""):
+
+ def retry(self, max_tries=5, wait_time=1, reason=""):
"""Retries and begin again from the beginning
:param max_tries: number of maximum retries
@@ -321,26 +404,28 @@ class Plugin(Base):
:param reason: reason for retrying, will be passed to fail if max_tries reached
"""
if 0 < max_tries <= self.retries:
- if not reason: reason = "Max retries reached"
- raise Fail(reason)
+ self.error(reason or _("Max retries reached"), "retry")
- self.wantReconnect = False
- self.setWait(wait_time)
- self.wait()
+ self.wait(wait_time, False)
self.retries += 1
raise Retry(reason)
+
def invalidCaptcha(self):
+ self.logError(_("Invalid captcha"))
if self.cTask:
self.cTask.invalid()
+
def correctCaptcha(self):
+ self.logInfo(_("Correct captcha"))
if self.cTask:
self.cTask.correct()
+
def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg',
- result_type='textual'):
+ result_type='textual', timeout=290):
""" Loads a captcha and decrypts it with ocr, plugin, user input
:param url: url of captcha image
@@ -359,9 +444,9 @@ class Plugin(Base):
img = self.load(url, get=get, post=post, cookies=cookies)
id = ("%.2f" % time())[-6:].replace(".", "")
- temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb")
- temp_file.write(img)
- temp_file.close()
+
+ with open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") as tmpCaptcha:
+ tmpCaptcha.write(img)
has_plugin = self.__name__ in self.core.pluginManager.ocrPlugins
@@ -372,20 +457,21 @@ class Plugin(Base):
if Ocr and not forceUser:
sleep(randint(3000, 5000) / 1000.0)
- if self.pyfile.abort: raise Abort
+ if self.pyfile.abort:
+ self.abort()
ocr = Ocr()
- result = ocr.get_captcha(temp_file.name)
+ result = ocr.get_captcha(tmpCaptcha.name)
else:
captchaManager = self.core.captchaManager
- task = captchaManager.newTask(img, imgtype, temp_file.name, result_type)
+ task = captchaManager.newTask(img, imgtype, tmpCaptcha.name, result_type)
self.cTask = task
- captchaManager.handleCaptcha(task)
+ captchaManager.handleCaptcha(task, timeout)
while task.isWaiting():
if self.pyfile.abort:
captchaManager.removeTask(task)
- raise Abort
+ self.abort()
sleep(1)
captchaManager.removeTask(task)
@@ -395,21 +481,21 @@ class Plugin(Base):
elif task.error:
self.fail(task.error)
elif not task.result:
- self.fail(_("No captcha result obtained in appropiate time by any of the plugins."))
+ self.fail(_("No captcha result obtained in appropiate time by any of the plugins"))
result = task.result
- self.logDebug("Received captcha result: %s" % str(result))
+ self.logDebug("Received captcha result: %s" % result)
if not self.core.debug:
try:
- remove(temp_file.name)
+ remove(tmpCaptcha.name)
except:
pass
return result
- def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False):
+ def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True):
"""Load content at url and returns it
:param url:
@@ -417,36 +503,44 @@ class Plugin(Base):
:param post:
:param ref:
:param cookies:
- :param just_header: if True only the header will be retrieved and returned as dict
+ :param just_header: If True only the header will be retrieved and returned as dict
:param decode: Wether to decode the output according to http header, should be True in most cases
+ :param follow_location: If True follow location else not
+ :param save_cookies: If True saves received cookies else discard them
:return: Loaded content
"""
- if self.pyfile.abort: raise Abort
- #utf8 vs decode -> please use decode attribute in all future plugins
- if type(url) == unicode:
- url = str(url) # encode('utf8')
+ if self.pyfile.abort:
+ self.abort()
+
+ if not url:
+ self.fail(_("No url given"))
- res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode)
+ if type(url) == unicode: # utf8 vs decode -> please use decode attribute in all future plugins
+ url = str(url) #: encode('utf8')
+
+ url = url.strip()
if self.core.debug:
- from inspect import currentframe
+ self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])
- frame = currentframe()
- if not exists(join("tmp", self.__name__)):
- makedirs(join("tmp", self.__name__))
+ res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies)
- f = open(
- join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
- , "wb")
- del frame # delete the frame or it wont be cleaned
+ if self.core.debug:
+ from inspect import currentframe
+ frame = currentframe()
+ framefile = save_join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
try:
- tmp = res.encode("utf8")
- except:
- tmp = res
+ if not exists(join("tmp", self.__name__)):
+ makedirs(join("tmp", self.__name__))
- f.write(tmp)
- f.close()
+ with open(framefile, "wb") as f:
+ del frame #: delete the frame or it wont be cleaned
+ if decode:
+ res = res.encode('utf-8')
+ f.write(res)
+ except IOError, e:
+ self.logError(e)
if just_header:
#parse header
@@ -456,7 +550,7 @@ class Plugin(Base):
if not line or ":" not in line: continue
key, none, value = line.partition(":")
- key = key.lower().strip()
+ key = key.strip().lower()
value = value.strip()
if key in header:
@@ -470,6 +564,7 @@ class Plugin(Base):
return res
+
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
"""Downloads the content at url to download folder
@@ -482,6 +577,19 @@ class Plugin(Base):
the filename will be changed if needed
:return: The location where the file was saved
"""
+ if self.pyfile.abort:
+ self.abort()
+
+ if not url:
+ self.fail(_("No url given"))
+
+ if type(url) == unicode:
+ url = str(url)
+
+ url = url.strip()
+
+ if self.core.debug:
+ self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])
self.checkForSameFiles()
@@ -492,16 +600,16 @@ class Plugin(Base):
location = safe_join(download_folder, self.pyfile.package().folder)
if not exists(location):
- makedirs(location, int(self.config['permission']['folder'], 8))
+ try:
+ makedirs(location, int(self.core.config['permission']['folder'], 8))
- if self.config['permission']['change_dl'] and os.name != "nt":
- try:
+ if self.core.config['permission']['change_dl'] and os.name != "nt":
uid = getpwnam(self.config['permission']['user'])[2]
gid = getgrnam(self.config['permission']['group'])[2]
-
chown(location, uid, gid)
- except Exception, e:
- self.logWarning(_("Setting User and Group failed: %s") % str(e))
+
+ except Exception, e:
+ self.fail(e)
# convert back to unicode
location = fs_decode(location)
@@ -518,28 +626,35 @@ class Plugin(Base):
finally:
self.pyfile.size = self.req.size
- if disposition and newname and newname != name: #triple check, just to be sure
- self.logInfo("%(name)s saved as %(newname)s" % {"name": name, "newname": newname})
- self.pyfile.name = newname
- filename = join(location, newname)
+ if newname:
+ newname = urlparse(newname).path.split("/")[-1]
+
+ if disposition and newname != name:
+ self.logInfo(_("%(name)s saved as %(newname)s") % {"name": name, "newname": newname})
+ self.pyfile.name = newname
+ filename = join(location, newname)
fs_filename = fs_encode(filename)
- if self.config['permission']['change_file']:
- chmod(fs_filename, int(self.config['permission']['file'], 8))
+ if self.core.config['permission']['change_file']:
+ try:
+ chmod(fs_filename, int(self.core.config['permission']['file'], 8))
+ except Exception, e:
+ self.logWarning(_("Setting file mode failed"), e)
- if self.config['permission']['change_dl'] and os.name != "nt":
+ if self.core.config['permission']['change_dl'] and os.name != "nt":
try:
uid = getpwnam(self.config['permission']['user'])[2]
gid = getgrnam(self.config['permission']['group'])[2]
-
chown(fs_filename, uid, gid)
+
except Exception, e:
- self.logWarning(_("Setting User and Group failed: %s") % str(e))
+ self.logWarning(_("Setting User and Group failed"), e)
self.lastDownload = filename
return self.lastDownload
+
def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0):
""" checks the content of the last downloaded file, re match is saved to `lastCheck`
@@ -551,7 +666,8 @@ class Plugin(Base):
:return: dictionary key of the first rule that matched
"""
lastDownload = fs_encode(self.lastDownload)
- if not exists(lastDownload): return None
+ if not exists(lastDownload):
+ return None
size = stat(lastDownload)
size = size.st_size
@@ -559,9 +675,10 @@ class Plugin(Base):
if api_size and api_size <= size: return None
elif size > max_size and not read_size: return None
self.logDebug("Download Check triggered")
- f = open(lastDownload, "rb")
- content = f.read(read_size if read_size else -1)
- f.close()
+
+ with open(lastDownload, "rb") as f:
+ content = f.read(read_size if read_size else -1)
+
#produces encoding errors, better log to other file in the future?
#self.logDebug("Content: %s" % content)
for name, rule in rules.iteritems():
@@ -609,7 +726,7 @@ class Plugin(Base):
if starting and self.config['download']['skip_existing'] and exists(location):
size = os.stat(location).st_size
if size >= self.pyfile.size:
- raise SkipDownload("File exists.")
+ raise SkipDownload("File exists")
pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
if pyfile:
@@ -618,14 +735,18 @@ class Plugin(Base):
self.logDebug("File %s not skipped, because it does not exists." % self.pyfile.name)
+
def clean(self):
""" clean everything and remove references """
if hasattr(self, "pyfile"):
del self.pyfile
+
if hasattr(self, "req"):
self.req.close()
del self.req
+
if hasattr(self, "thread"):
del self.thread
+
if hasattr(self, "html"):
del self.html
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-23 22:15:10 +0100