# -*- coding: utf-8 -*-
"""
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License,
or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, see <http://www.gnu.org/licenses/>.
@author: RaNaN, spoob, mkaay
"""
from time import time
from time import sleep
from random import randint
import os
from os import remove
from os import makedirs
from os import chmod
from os import stat
from os import name as os_name
from os.path import exists, join
if os.name != "nt":
from os import chown
from pwd import getpwnam
from grp import getgrnam
from itertools import islice
from module.utils import save_join, removeChars
def chunks(iterable, size):
it = iter(iterable)
item = list(islice(it, size))
while item:
yield item
item = list(islice(it, size))
class Abort(Exception):
""" raised when aborted """
class Fail(Exception):
""" raised when failed """
class Reconnect(Exception):
""" raised when reconnected """
class Retry(Exception):
""" raised when start again from beginning """
class SkipDownload(Exception):
""" raised when download should be skipped """
class Plugin(object):
__name__ = "Plugin"
__version__ = "0.4"
__pattern__ = None
__type__ = "hoster"
__config__ = [("name", "type", "desc", "default")]
__description__ = """Base Plugin"""
__author_name__ = ("RaNaN", "spoob", "mkaay")
__author_mail__ = ("RaNaN@pyload.org", "spoob@pyload.org", "mkaay@mkaay.de")
def __init__(self, pyfile):
self.config = pyfile.m.core.config
self.core = pyfile.m.core
self.wantReconnect = False
self.multiDL = True #: enables simultaneous processing of multiple downloads
self.limitDL = 0
self.chunkLimit = 1
self.resumeDownload = False
self.waitUntil = 0 #: time() + wait in seconds
self.waiting = False
self.ocr = None #captcha reader instance
self.account = pyfile.m.core.accountManager.getAccountPlugin(self.__name__) #: account handler instance, see :py:class:`Account`
self.premium = False
self.user = None
if self.account and not self.account.canUse(): self.account = None
if self.account:
self.user, data = self.account.selectAccount()
#: Browser instance, see `network.Browser`
self.req = self.account.getAccountRequest(self.user)
self.chunkLimit = -1 #: chunk limit, -1 for unlimited
self.resumeDownload = True #: enables resume (will be ignored if server dont accept chunks)
self.multiDL = True #every hoster with account should provide multiple downloads
self.premium = self.account.isPremium(self.user) #: premium status
else:
self.req = pyfile.m.core.requestFactory.getRequest(self.__name__)
self.log = pyfile.m.core.log
self.pyfile = pyfile
self.thread = None # holds thread in future
self.lastDownload = "" #: location where the last call to download was saved
self.lastCheck = None #: re match of the last call to `checkDownload`
self.js = self.core.js #: js engine, see `JsEngine`
self.cTask = None #captcha task
self.html = None #some plugins store html code here
self.init()
def getChunkCount(self):
if self.chunkLimit <= 0:
return self.config["download"]["chunks"]
return min(self.config["download"]["chunks"], self.chunkLimit)
def __call__(self):
return self.__name__
def init(self):
"""
initialize the plugin (in addition to `__init__`)
"""
pass
def setup(self):
""" setup for enviroment and other things, called before downloading (possibly more than one time)"""
pass
def preprocessing(self, thread):
""" handles important things to do before starting """
self.thread = thread
if self.account:
self.account.checkLogin(self.user)
else:
self.req.clearCookies()
self.setup()
self.pyfile.setStatus("starting")
return self.process(self.pyfile)
#----------------------------------------------------------------------
def process(self, pyfile):
"""the 'main' method of every plugin, you **have to** overwrite it"""
raise NotImplementedError
def resetAccount(self):
""" dont use account and retry download """
self.account = None
self.req = self.core.requestFactory.getRequest(self.__name__)
self.retry()
def checksum(self, local_file=None):
"""
return codes:
0 - checksum ok
1 - checksum wrong
5 - can't get checksum
10 - not implemented
20 - unknown error
"""
#@TODO checksum check hook
return True, 10
def setConf(self, option, value):
""" sets a config value """
self.config.setPlugin(self.__name__, option, value)
def getConf(self, option):
""" gets a config value """
return self.config.getPlugin(self.__name__, option)
def setConfig(self, option, value):
""" sets a config value """
self.setConf(option, value)
def getConfig(self, option):
""" gets a config value """
return self.getConf(option)
def setWait(self, seconds, reconnect=False):
"""Set a specific wait time later used with `wait`
:param seconds: wait time in seconds
:param reconnect: True if a reconnect would avoid wait time
"""
if reconnect:
self.wantReconnect = True
self.pyfile.waitUntil = time() + int(seconds)
def wait(self):
""" waits the time previously set """
self.waiting = True
self.pyfile.setStatus("waiting")
while self.pyfile.waitUntil > time():
self.thread.m.reconnecting.wait(2)
if self.pyfile.abort: raise Abort
if self.thread.m.reconnecting.isSet():
self.waiting = False
self.wantReconnect = False
raise Reconnect
self.waiting = False
self.pyfile.setStatus("starting")
def fail(self, reason):
""" fail and give reason """
raise Fail(reason)
def offline(self):
""" fail and indicate file is offline """
raise Fail("offline")
def tempOffline(self):
""" fail and indicates file ist temporary offline, the core may take consequences """
raise Fail("temp. offline")
def retry(self):
""" begin again from the beginning """
raise Retry
def invalidCaptcha(self):
if self.cTask:
self.cTask.invalid()
def correctCaptcha(self):
if self.cTask:
self.cTask.correct()
def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', result_type='textual'):
""" Loads a captcha and decrypts it with ocr, plugin, user input
:param url: url of captcha image
:param get: get part for request
:param post: post part for request
:param cookies: True if cookies should be enabled
:param forceUser: if True, ocr is not used
:param imgtype: Type of the Image
:param result_type: 'textual' if text is written on the captcha\
or 'positional' for captcha where the user have to click\
on a specific region on the captcha
:return: result of decrypting
"""
img = self.load(url, get=get, post=post, cookies=cookies)
id = ("%.2f" % time())[-6:].replace(".","")
temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb")
temp_file.write(img)
temp_file.close()
has_plugin = self.core.pluginManager.captchaPlugins.has_key(self.__name__)
if self.core.captcha:
Ocr = self.core.pluginManager.getCaptchaPlugin(self.__name__)
else:
Ocr = None
if Ocr and not forceUser:
sleep(randint(3000, 5000) / 1000.0)
if self.pyfile.abort: raise Abort
ocr = Ocr()
result = ocr.get_captcha(temp_file.name)
else:
captchaManager = self.core.captchaManager
task = captchaManager.newTask(img, imgtype, temp_file.name, result_type)
self.cTask = task
captchaManager.handleCaptcha(task)
while task.isWaiting():
if self.pyfile.abort:
captchaManager.removeTask(task)
raise Abort
sleep(1)
captchaManager.removeTask(task)
if task.error and has_plugin: #ignore default error message since the user could use OCR
self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting"))
elif task.error:
self.fail(task.error)
elif not task.result:
self.fail(_("No captcha result obtained in appropiate time by any of the plugins."))
result = task.result
self.log.debug("Received captcha result: %s" % str(result))
if not self.core.debug:
try:
remove(temp_file.name)
except:
pass
return result
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, utf8=False, decode=False):
"""Load content at url and returns it
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param just_header: if True only the header will be retrieved and returned as dict
:param utf8: Deprecated
:param decode: Wether to decode the output according to http header, should be True in most cases
:return: Loaded content
"""
if self.pyfile.abort: raise Abort
#utf8 vs decode -> please use decode attribute in all future plugins
res = self.req.load(url, get, post, ref, cookies, just_header, decode=utf8 or decode)
if self.core.debug:
from inspect import currentframe
frame = currentframe()
if not exists(join("tmp", self.__name__)):
makedirs(join("tmp", self.__name__))
f = open(
join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
, "wb")
del frame # delete the frame or it wont be cleaned
try:
tmp = res.encode("utf8")
except:
tmp = res
f.write(tmp)
f.close()
if just_header:
#parse header
header = {"code": self.req.code}
for line in res.splitlines():
line = line.strip()
if not line or ":" not in line: continue
key, none, value = line.partition(":")
key = key.lower().strip()
value = value.strip()
if header.has_key(key):
if type(header[key]) == list:
header[key].append(value)
else:
header[key] = [header[key], value]
else:
header[key] = value
res = header
return res
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
"""Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"], 8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception, e:
self.log.warning(_("Setting User and Group failed: %s") % str(e))
name = self.pyfile.name
if os_name == 'nt':
#delete illegal characters
name = removeChars(name, '/\\?%*:|"<>')
else:
name = removeChars(name, '/\\"')
filename = save_join(location, name)
try:
newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies,
chunks=self.getChunkCount(), resume=self.resumeDownload,
progressNotify=self.pyfile.progress.setValue, disposition=disposition)
finally:
self.pyfile.size = self.req.size
if disposition and newname and newname != name: #triple check, just to be sure
self.log.info("%(name)s saved as %(newname)s" % {"name": name, "newname": newname})
self.pyfile.name = newname
filename = save_join(location, newname)
if self.core.config["permission"]["change_file"]:
chmod(filename, int(self.core.config["permission"]["file"], 8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(filename, uid, gid)
except Exception, e:
self.log.warning(_("Setting User and Group failed: %s") % str(e))
self.lastDownload = filename
return self.lastDownload
def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0):
""" checks the content of the last downloaded file, re match is saved to `lastCheck`
:param rules: dict with names and rules to match (compiled regexp or strings)
:param api_size: expected file size
:param max_size: if the file is larger then it wont be checked
:param delete: delete if matched
:param read_size: amount of bytes to read from files larger then max_size
:return: dictionary key of the first rule that matched
"""
if not exists(self.lastDownload): return None
size = stat(self.lastDownload)
size = size.st_size
if api_size and api_size <= size: return None
elif size > max_size and not read_size: return None
self.log.debug("Download Check triggered")
f = open(self.lastDownload, "rb")
content = f.read(read_size if read_size else -1)
f.close()
#produces encoding errors, better log to other file in the future?
#self.log.debug("Content: %s" % content)
for name, rule in rules.iteritems():
if type(rule) in (str, unicode):
if rule in content:
if delete:
remove(self.lastDownload)
return name
elif hasattr(rule, "search"):
m = rule.search(content)
if m:
if delete:
remove(self.lastDownload)
self.lastCheck = m
return name
def getPassword(self):
""" get the password the user provided in the package"""
password = self.pyfile.package().password
if not password: return ""
return password
def checkForSameFiles(self, starting=False):
""" checks if same file was/is downloaded within same package and raise exception """
pack = self.pyfile.package()
cache = self.core.files.cache.values()
for pyfile in cache:
if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
if pyfile.status in (0, 12): #finished or downloading
raise SkipDownload(pyfile.pluginname)
elif pyfile.status in (5, 7) and starting: #a download is waiting and was appenrently started before
raise SkipDownload(pyfile.pluginname)
#TODO check same packagenames
pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.packageid, self.pyfile.name)
if pyfile:
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, pack.folder)
if exists(save_join(location, self.pyfile.name)):
raise SkipDownload(pyfile[0])
self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
#log functions
def logInfo(self, msg):
self.log.info("%s: %s" % (self.__name__, msg))
def logWarning(self, msg):
self.log.warning("%s: %s" % (self.__name__, msg))
def logError(self, msg):
self.log.error("%s: %s" % (self.__name__, msg))
def logDebug(self, msg):
self.log.debug("%s: %s" % (self.__name__, msg))
def clean(self):
""" clean everything and remove references """
if hasattr(self, "pyfile"):
del self.pyfile
if hasattr(self, "req"):
self.req.close()
del self.req
if hasattr(self, "thread"):
del self.thread
if hasattr(self, "html"):
del self.html