diff options
Diffstat (limited to 'module/plugins')
| -rw-r--r-- | module/plugins/hooks/ExtractArchive.py | 20 | ||||
| -rw-r--r-- | module/plugins/hoster/BasePlugin.py | 78 | ||||
| -rw-r--r-- | module/plugins/hoster/DepositfilesCom.py | 46 | ||||
| -rw-r--r-- | module/plugins/hoster/HellshareCz.py | 4 | ||||
| -rw-r--r-- | module/plugins/hoster/HotfileCom.py | 9 | ||||
| -rw-r--r-- | module/plugins/internal/UnRar.py | 6 |
6 files changed, 106 insertions, 57 deletions
diff --git a/module/plugins/hooks/ExtractArchive.py b/module/plugins/hooks/ExtractArchive.py index 54dd6ff2d..0a70da417 100644 --- a/module/plugins/hooks/ExtractArchive.py +++ b/module/plugins/hooks/ExtractArchive.py @@ -3,8 +3,8 @@ import sys import os -from os import remove, chmod -from os.path import exists, basename, isfile, isdir +from os import remove, chmod, makedirs +from os.path import exists, basename, isfile, isdir, join from traceback import print_exc from copy import copy @@ -50,7 +50,7 @@ if os.name != "nt": from grp import getgrnam from module.plugins.Hook import Hook, threaded, Expose -from module.utils import save_join +from module.utils import save_join, fs_encode class ArchiveError(Exception): @@ -77,6 +77,7 @@ class ExtractArchive(Hook): ("overwrite", "bool", "Overwrite files", True), ("passwordfile", "file", "password file", "unrar_passwords.txt"), ("deletearchive", "bool", "Delete archives when done", False), + ("subfolder", "bool", "Create subfolder for each package", False), ("destination", "folder", "Extract files to", ""), ("queue", "bool", "Wait for all downloads to be fninished", True), ("renice", "int", "CPU Priority", 0), ] @@ -151,7 +152,7 @@ class ExtractArchive(Hook): #iterate packages -> plugins -> targets for pid in ids: p = self.core.files.getPackage(pid) - self.logInfo(_("Extract package %s") % p.name) + self.logInfo(_("Check package %s") % p.name) if not p: continue # determine output folder @@ -159,8 +160,15 @@ class ExtractArchive(Hook): # force trailing slash if self.getConfig("destination") and self.getConfig("destination").lower() != "none": - if exists(self.getConfig("destination")): - out = save_join(self.getConfig("destination"), "") + + out = save_join(dl, p.folder, self.getConfig("destination"), "") + #relative to package folder if destination is relative, otherwise absolute path overwrites them + + if self.getConf("subfolder"): + out = join(out, fs_encode(p.folder)) + + if not exists(out): + makedirs(out) files_ids = [(save_join(dl, p.folder, x["name"]), x["id"]) for x in p.getChildren().itervalues()] diff --git a/module/plugins/hoster/BasePlugin.py b/module/plugins/hoster/BasePlugin.py index ed11c378b..15e35ce24 100644 --- a/module/plugins/hoster/BasePlugin.py +++ b/module/plugins/hoster/BasePlugin.py @@ -1,13 +1,13 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - -from module.plugins.Hoster import Hoster -from module.utils import html_unescape - from urlparse import urlparse from re import search from urllib import unquote +from module.network.HTTPRequest import BadHeader +from module.plugins.Hoster import Hoster +from module.utils import html_unescape, removeChars + class BasePlugin(Hoster): __name__ = "BasePlugin" __type__ = "hoster" @@ -41,31 +41,49 @@ class BasePlugin(Hoster): # # return if pyfile.url.startswith("http"): - header = self.load(pyfile.url, just_header = True) - #self.logDebug(header) - - if 'location' in header: - self.logDebug("Location: " + header['location']) - url = unquote(header['location']) - else: - url = pyfile.url - - name = html_unescape(urlparse(url).path.split("/")[-1]) - - if 'content-disposition' in header: - self.logDebug("Content-Disposition: " + header['content-disposition']) - m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", header['content-disposition']) - if m: - disp = m.groupdict() - self.logDebug(disp) - if not disp['enc']: disp['enc'] = 'utf-8' - name = disp['name'].replace('"', "").replace("'", "").replace(";", "").strip() - name = unicode(unquote(name), disp['enc']) - - if not name: self.offline() - pyfile.name = name - self.logDebug("Filename: %s" % pyfile.name) - self.download(url, disposition=True) + + try: + self.downloadFile(pyfile) + except BadHeader, e: + if e.code in (401, 403): + self.logDebug("Auth required") + + pwd = pyfile.package().password.strip() + if ":" not in pwd: + self.fail(_("Authorization required (username:password)")) + + self.req.addAuth(pwd) + self.downloadFile(pyfile) + else: + raise else: - self.fail("No Plugin matched and not a downloadable url.")
\ No newline at end of file + self.fail("No Plugin matched and not a downloadable url.") + + + def downloadFile(self, pyfile): + header = self.load(pyfile.url, just_header = True) + #self.logDebug(header) + + if 'location' in header: + self.logDebug("Location: " + header['location']) + url = unquote(header['location']) + else: + url = pyfile.url + + name = html_unescape(urlparse(url).path.split("/")[-1]) + + if 'content-disposition' in header: + self.logDebug("Content-Disposition: " + header['content-disposition']) + m = search("filename(?P<type>=|\*=(?P<enc>.+)'')(?P<name>.*)", header['content-disposition']) + if m: + disp = m.groupdict() + self.logDebug(disp) + if not disp['enc']: disp['enc'] = 'utf-8' + name = removeChars(disp['name'], "\"';").strip() + name = unicode(unquote(name), disp['enc']) + + if not name: name = url + pyfile.name = name + self.logDebug("Filename: %s" % pyfile.name) + self.download(url, disposition=True)
\ No newline at end of file diff --git a/module/plugins/hoster/DepositfilesCom.py b/module/plugins/hoster/DepositfilesCom.py index 1fba3bad5..81e6aa4d6 100644 --- a/module/plugins/hoster/DepositfilesCom.py +++ b/module/plugins/hoster/DepositfilesCom.py @@ -5,6 +5,7 @@ import re import urllib from module.plugins.Hoster import Hoster from module.network.RequestFactory import getURL +from module.plugins.ReCaptcha import ReCaptcha def getInfo(urls): result = [] @@ -29,13 +30,15 @@ class DepositfilesCom(Hoster): __name__ = "DepositfilesCom" __type__ = "hoster" __pattern__ = r"http://[\w\.]*?depositfiles\.com(/\w{1,3})?/files/[\w]+" - __version__ = "0.33" + __version__ = "0.34" __description__ = """Depositfiles.com Download Hoster""" __author_name__ = ("spoob", "zoidberg") __author_mail__ = ("spoob@pyload.org", "zoidberg@mujmail.cz") FILE_INFO_PATTERN = r'File name: <b title="([^"]+)">.*\s*<span class="nowrap">File size: <b>([0-9.]+) (KB|MB|GB)</b>' FILE_OFFLINE_PATTERN = r'<span class="html_download_api-not_exists"></span>' + RECAPTCHA_PATTERN = r"Recaptcha.create\('([^']+)', this\);" + DOWNLOAD_LINK_PATTERN = r'<form action="(http://.+?\.depositfiles.com/.+?)" method="get"' def setup(self): self.resumeDownload = self.multiDL = True if self.account else False @@ -96,27 +99,42 @@ class DepositfilesCom(Hoster): found = re.search(r"var fid = '(\w+)';", self.html) if not found: self.retry(wait_time=5) - fid = found.group(1) - self.logDebug ("FID: %s" % fid) + params = {'fid' : found.group(1)} + self.logDebug ("FID: %s" % params['fid']) - self.wait() - - #form = re.search(r"\$\('#download_container'\)\.load\('([^']+)", self.html) + captcha_key = None + found = re.search(self.RECAPTCHA_PATTERN, self.html) + if found: captcha_key = found.group(1) + self.logDebug ("CAPTCHA_KEY: %s" % captcha_key) - #self.html = self.load("http://depositfiles.com/"+ form.group(1)) + self.wait() + recaptcha = ReCaptcha(self) + + for i in range(5): + self.html = self.load("http://depositfiles.com/get_file.php", get = params) + if '<input type=button value="Continue" onclick="check_recaptcha' in self.html: + if not captcha_key: self.fail('Parse error (Captcha key)') + if 'response' in params: self.invalidCaptcha() + params['challenge'], params['response'] = recaptcha.challenge(captcha_key) + self.logDebug(params) + continue + + found = re.search(self.DOWNLOAD_LINK_PATTERN, self.html) + if found: + if 'response' in params: self.correctCaptcha() + link = urllib.unquote(found.group(1)) + self.logDebug ("LINK: %s" % link) + break + else: + self.fail('Parse error (Download link)') + else: + self.fail('No valid captcha response received') - self.html = self.load("http://depositfiles.com/get_file.php?fid=" + fid) - link = urllib.unquote(re.search('<form action="(http://.+?\.depositfiles.com/.+?)" method="get"', self.html).group(1)) - self.logDebug ("LINK: %s" % link) try: self.download(link) except: self.retry(wait_time = 60) - #wait_time = int(re.search(r'<span id="download_waiter_remain">(.*?)</span>', self.html).group(1)) - #self.setWait(wait_time) - #self.log.debug("DepositFiles.com: Waiting %d seconds." % wait_time) - def handlePremium(self): link = urllib.unquote(re.search('<div id="download_url">\s*<a href="(http://.+?\.depositfiles.com/.+?)"', self.html).group(1)) self.download(link)
\ No newline at end of file diff --git a/module/plugins/hoster/HellshareCz.py b/module/plugins/hoster/HellshareCz.py index 7986ecf11..8c90e8099 100644 --- a/module/plugins/hoster/HellshareCz.py +++ b/module/plugins/hoster/HellshareCz.py @@ -48,7 +48,7 @@ class HellshareCz(Hoster): __name__ = "HellshareCz" __type__ = "hoster" __pattern__ = r"http://(.*\.)*hellshare\.(cz|com|sk|hu)/.*" - __version__ = "0.70" + __version__ = "0.71" __description__ = """Hellshare.cz""" __author_name__ = ("zoidberg") @@ -97,7 +97,7 @@ class HellshareCz(Hoster): def handleFree(self): # hellshare is very generous - if 'You exceeded your today's limit for free download. You can download only 1 files per 24 hours.' in self.html: + if "You exceeded your today's limit for free download. You can download only 1 files per 24 hours." in self.html: t = datetime.datetime.today().replace(hour=1, minute=0, second=0) + datetime.timedelta( days=1) - datetime.datetime.today() self.setWait(t.seconds, True) diff --git a/module/plugins/hoster/HotfileCom.py b/module/plugins/hoster/HotfileCom.py index 96927a9fc..9c056d899 100644 --- a/module/plugins/hoster/HotfileCom.py +++ b/module/plugins/hoster/HotfileCom.py @@ -37,6 +37,8 @@ class HotfileCom(Hoster): __author_name__ = ("sitacuisses","spoob","mkaay") __author_mail__ = ("sitacuisses@yhoo.de","spoob@pyload.org","mkaay@mkaay.de") + FILE_OFFLINE_PATTERN = r'File is removed' + def setup(self): self.html = [None, None] self.wantReconnect = False @@ -73,7 +75,10 @@ class HotfileCom(Hoster): if not self.account: self.downloadHTML() - + + if self.FILE_OFFLINE_PATTERN in self.html[0]: + self.offline() + self.setWait(self.getWaitTime()) self.wait() @@ -131,3 +136,5 @@ class HotfileCom(Hoster): self.wantReconnect = True return waittime return 65 + else: + self.fail("Don't know how long to wait. Cannot proceed.") diff --git a/module/plugins/internal/UnRar.py b/module/plugins/internal/UnRar.py index 7ba1d01ba..1943f69e0 100644 --- a/module/plugins/internal/UnRar.py +++ b/module/plugins/internal/UnRar.py @@ -18,16 +18,14 @@ """ import os +import re from os.path import join from glob import glob from subprocess import Popen, PIPE - from module.plugins.hooks.ExtractArchive import AbtractExtractor from module.utils import save_join, decode -import re - class UnRar(AbtractExtractor): __name__ = "UnRar" __version__ = "0.1" @@ -138,7 +136,7 @@ class UnRar(AbtractExtractor): def getDeleteFiles(self): if ".part" in self.file: - return glob(self.file.replace("0", "*").replace("1", "*")) + return glob(re.sub("(?<=\.part)([01]+)", "*", self.file, re.IGNORECASE)) return [self.file] def listContent(self): |