diff options
| -rw-r--r-- | module/PluginThread.py | 684 | ||||
| -rw-r--r-- | pyload/plugins/Plugin.py | 3 | ||||
| -rw-r--r-- | pyload/plugins/account/UptoboxCom.py | 3 | ||||
| -rw-r--r-- | pyload/plugins/hoster/EasybytezCom.py | 3 | ||||
| -rw-r--r-- | pyload/plugins/hoster/RarefileNet.py | 16 | ||||
| -rw-r--r-- | pyload/plugins/internal/SimpleCrypter.py | 47 | ||||
| -rw-r--r-- | pyload/plugins/internal/SimpleHoster.py | 129 | ||||
| -rw-r--r-- | pyload/plugins/internal/XFSAccount.py | 5 | ||||
| -rw-r--r-- | pyload/plugins/internal/XFSHoster.py | 152 |
9 files changed, 884 insertions, 158 deletions
diff --git a/module/PluginThread.py b/module/PluginThread.py new file mode 100644 index 000000000..051236c3e --- /dev/null +++ b/module/PluginThread.py @@ -0,0 +1,684 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: RaNaN +""" + +from Queue import Queue +from threading import Thread +from os import listdir, stat +from os.path import join +from time import sleep, time, strftime, gmtime +from traceback import print_exc, format_exc +from pprint import pformat +from sys import exc_info, exc_clear +from copy import copy +from types import MethodType + +from pycurl import error + +from PyFile import PyFile +from plugins.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload +from common.packagetools import parseNames +from utils import save_join +from Api import OnlineStatus + +class PluginThread(Thread): + """abstract base class for thread types""" + + #---------------------------------------------------------------------- + def __init__(self, manager): + """Constructor""" + Thread.__init__(self) + self.setDaemon(True) + self.m = manager #thread manager + + + def writeDebugReport(self, pyfile): + """ writes a + :return: + """ + + dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) + dump = self.getDebugDump(pyfile) + + try: + import zipfile + + zip = zipfile.ZipFile(dump_name, "w") + + for f in listdir(join("tmp", pyfile.pluginname)): + try: + # avoid encoding errors + zip.write(join("tmp", pyfile.pluginname, f), save_join(pyfile.pluginname, f)) + except: + pass + + info = zipfile.ZipInfo(save_join(pyfile.pluginname, "debug_Report.txt"), gmtime()) + info.external_attr = 0644 << 16L # change permissions + + zip.writestr(info, dump) + zip.close() + + if not stat(dump_name).st_size: + raise Exception("Empty Zipfile") + + except Exception, e: + self.m.log.debug("Error creating zip file: %s" % e) + + dump_name = dump_name.replace(".zip", ".txt") + f = open(dump_name, "wb") + f.write(dump) + f.close() + + self.m.core.log.info("Debug Report written to %s" % dump_name) + + def getDebugDump(self, pyfile): + dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( + self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version__, format_exc()) + + tb = exc_info()[2] + stack = [] + while tb: + stack.append(tb.tb_frame) + tb = tb.tb_next + + for frame in stack[1:]: + dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_lineno) + + for key, value in frame.f_locals.items(): + dump += "\t%20s = " % key + try: + dump += pformat(value) + "\n" + except Exception, e: + dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" + + del frame + + del stack #delete it just to be sure... + + dump += "\n\nPLUGIN OBJECT DUMP: \n\n" + + for name in dir(pyfile.plugin): + attr = getattr(pyfile.plugin, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" + + dump += "\nPYFILE OBJECT DUMP: \n\n" + + for name in dir(pyfile): + attr = getattr(pyfile, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += "<ERROR WHILE PRINTING VALUE> " + str(e) + "\n" + + if pyfile.pluginname in self.m.core.config.plugin: + dump += "\n\nCONFIG: \n\n" + dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" + + return dump + + def clean(self, pyfile): + """ set thread unactive and release pyfile """ + self.active = False + pyfile.release() + + +class DownloadThread(PluginThread): + """thread for downloading files from 'real' hoster plugins""" + + #---------------------------------------------------------------------- + def __init__(self, manager): + """Constructor""" + PluginThread.__init__(self, manager) + + self.queue = Queue() #: job queue + self.active = False + + self.start() + + #---------------------------------------------------------------------- + def run(self): + """run method""" + pyfile = None + + while True: + del pyfile + self.active = self.queue.get() + pyfile = self.active + + if self.active == "quit": + self.active = False + self.m.threads.remove(self) + return True + + try: + if not pyfile.hasPlugin(): + continue + #this pyfile was deleted while queueing + + pyfile.plugin.checkForSameFiles(starting=True) + self.m.log.info(_("Download starts: %s" % pyfile.name)) + + # start download + self.m.core.hookManager.downloadPreparing(pyfile) + pyfile.error = "" + pyfile.plugin.preprocessing(self) + + self.m.log.info(_("Download finished: %s") % pyfile.name) + self.m.core.hookManager.downloadFinished(pyfile) + self.m.core.files.checkPackageFinished(pyfile) + + except NotImplementedError: + self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + pyfile.setStatus("failed") + pyfile.error = "Plugin does not work" + self.clean(pyfile) + continue + + except Abort: + try: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + except: + pass + + pyfile.setStatus("aborted") + + if self.m.core.debug: + print_exc() + + self.clean(pyfile) + continue + + except Reconnect: + self.queue.put(pyfile) + #pyfile.req.clearCookies() + + while self.m.reconnecting.isSet(): + sleep(0.5) + + continue + + except Retry, e: + reason = e.args[0] + self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) + self.queue.put(pyfile) + continue + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + pyfile.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % pyfile.name) + elif msg == "temp. offline": + pyfile.setStatus("temp. offline") + self.m.log.warning(_("Download is temporary offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + + if self.m.core.debug: + print_exc() + + self.m.core.hookManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + except error, e: + if len(e.args) == 2: + code, msg = e.args + else: + code = 0 + msg = e.args + + self.m.log.debug("pycurl exception %s: %s" % (code, msg)) + + if code in (7, 18, 28, 52, 56): + self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) + wait = time() + 60 + + pyfile.waitUntil = wait + pyfile.setStatus("waiting") + while time() < wait: + sleep(1) + if pyfile.abort: + break + + if pyfile.abort: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + self.clean(pyfile) + else: + self.queue.put(pyfile) + + continue + + else: + pyfile.setStatus("failed") + self.m.log.error("pycurl error %s: %s" % (code, msg)) + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + self.m.core.hookManager.downloadFailed(pyfile) + + self.clean(pyfile) + continue + + except SkipDownload, e: + pyfile.setStatus("skipped") + + self.m.log.info( + _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) + + self.clean(pyfile) + + self.m.core.files.checkPackageFinished(pyfile) + + self.active = False + self.m.core.files.save() + + continue + + + except Exception, e: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) + + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + self.m.core.hookManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + finally: + self.m.core.files.save() + pyfile.checkIfProcessed() + exc_clear() + + + #pyfile.plugin.req.clean() + + self.active = False + pyfile.finishIfDone() + self.m.core.files.save() + + + def put(self, job): + """assing job to thread""" + self.queue.put(job) + + + def stop(self): + """stops the thread""" + self.put("quit") + + +class DecrypterThread(PluginThread): + """thread for decrypting""" + + def __init__(self, manager, pyfile): + """constructor""" + PluginThread.__init__(self, manager) + + self.active = pyfile + manager.localThreads.append(self) + + pyfile.setStatus("decrypting") + + self.start() + + def getActiveFiles(self): + return [self.active] + + def run(self): + """run method""" + + pyfile = self.active + retry = False + + try: + self.m.log.info(_("Decrypting starts: %s") % pyfile.name) + pyfile.error = "" + pyfile.plugin.preprocessing(self) + + except NotImplementedError: + self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + return + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + pyfile.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + + if self.m.core.debug: + print_exc() + return + + except Abort: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + if self.m.core.debug: + print_exc() + return + + except Retry: + self.m.log.info(_("Retrying %s") % pyfile.name) + retry = True + return self.run() + + except Exception, e: + pyfile.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) + + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + return + + finally: + if not retry: + pyfile.release() + self.active = False + self.m.core.files.save() + self.m.localThreads.remove(self) + exc_clear() + + if not retry: + pyfile.delete() + + +class HookThread(PluginThread): + """thread for hooks""" + + #---------------------------------------------------------------------- + def __init__(self, m, function, args, kwargs): + """Constructor""" + PluginThread.__init__(self, m) + + self.f = function + self.args = args + self.kwargs = kwargs + + self.active = [] + + m.localThreads.append(self) + + self.start() + + def getActiveFiles(self): + return self.active + + def addActive(self, pyfile): + """ Adds a pyfile to active list and thus will be displayed on overview""" + if pyfile not in self.active: + self.active.append(pyfile) + + def finishFile(self, pyfile): + if pyfile in self.active: + self.active.remove(pyfile) + + pyfile.finishIfDone() + + def run(self): + try: + try: + self.kwargs["thread"] = self + self.f(*self.args, **self.kwargs) + except TypeError, e: + #dirty method to filter out exceptions + if "unexpected keyword argument 'thread'" not in e.args[0]: + raise + + del self.kwargs["thread"] + self.f(*self.args, **self.kwargs) + finally: + local = copy(self.active) + for x in local: + self.finishFile(x) + + self.m.localThreads.remove(self) + + +class InfoThread(PluginThread): + def __init__(self, manager, data, pid=-1, rid=-1, add=False): + """Constructor""" + PluginThread.__init__(self, manager) + + self.data = data + self.pid = pid # package id + # [ .. (name, plugin) .. ] + + self.rid = rid #result id + self.add = add #add packages instead of return result + + self.cache = [] #accumulated data + + self.start() + + def run(self): + """run method""" + + plugins = {} + container = [] + + for url, plugin in self.data: + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + + # filter out container plugins + for name in self.m.core.pluginManager.containerPlugins: + if name in plugins: + container.extend([(name, url) for url in plugins[name]]) + + del plugins[name] + + #directly write to database + if self.pid > -1: + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) + self.m.core.files.save() + + elif self.add: + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateCache(pluginname, result) + + packs = parseNames([(name, url) for name, x, y, url in self.cache]) + + self.m.log.debug("Fetched and generated %d packages" % len(packs)) + + for k, v in packs: + self.m.core.api.addPackage(k, v) + + #empty cache + del self.cache[:] + + else: #post the results + + + for name, url in container: + #attach container content + try: + data = self.decryptContainer(name, url) + except: + print_exc() + self.m.log.error("Could not decrypt container.") + data = [] + + for url, plugin in data: + if plugin in plugins: + plugins[plugin].append(url) + else: + plugins[plugin] = [url] + + self.m.infoResults[self.rid] = {} + + for pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) + + #force to process cache + if self.cache: + self.updateResult(pluginname, [], True) + + else: + #generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateResult(pluginname, result, True) + + self.m.infoResults[self.rid]["ALL_INFO_FETCHED"] = {} + + self.m.timestamp = time() + 5 * 60 + + + def updateDB(self, plugin, result): + self.m.core.files.updateFileInfo(result, self.pid) + + def updateResult(self, plugin, result, force=False): + #parse package name and generate result + #accumulate results + + self.cache.extend(result) + + if len(self.cache) >= 20 or force: + #used for package generating + tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) + for name, size, status, url in self.cache] + + data = parseNames(tmp) + result = {} + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + self.m.setInfoResults(self.rid, result) + + self.cache = [] + + def updateCache(self, plugin, result): + self.cache.extend(result) + + def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): + try: + result = [] #result loaded from cache + process = [] #urls to process + for url in urls: + if url in self.m.infoCache: + result.append(self.m.infoCache[url]) + else: + process.append(url) + + if result: + self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) + cb(pluginname, result) + + if process: + self.m.log.debug("Run Info Fetching for %s" % pluginname) + for result in plugin.getInfo(process): + #result = [ .. (name, size, status, url) .. ] + if not type(result) == list: + result = [result] + + for res in result: #: why don't assign res dict directly? + self.m.infoCache[res[3]] = res + + cb(pluginname, result) + + self.m.log.debug("Finished Info Fetching for %s" % pluginname) + except Exception, e: + self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % + {"name": pluginname, "err": str(e)}) + if self.m.core.debug: + print_exc() + + # generate default results + if err: + result = [(url, 0, 3, url) for url in urls] + cb(pluginname, result) + + + def decryptContainer(self, plugin, url): + data = [] + # only works on container plugins + + self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) + + # dummy pyfile + pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) + + pyfile.initPlugin() + + # little plugin lifecycle + try: + pyfile.plugin.setup() + pyfile.plugin.loadToDisk() + pyfile.plugin.decrypt(pyfile) + pyfile.plugin.deleteTmp() + + for pack in pyfile.plugin.packages: + pyfile.plugin.urls.extend(pack[1]) + + data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) + + self.m.log.debug("Got %d links." % len(data)) + + except Exception, e: + self.m.log.debug("Pre decrypting error: %s" % str(e)) + finally: + pyfile.release() + + return data diff --git a/pyload/plugins/Plugin.py b/pyload/plugins/Plugin.py index a81fbef33..cf4debd37 100644 --- a/pyload/plugins/Plugin.py +++ b/pyload/plugins/Plugin.py @@ -16,7 +16,8 @@ from itertools import islice from traceback import print_exc from urlparse import urlparse -from pyload.utils import encode, fs_decode, fs_encode, html_unescape, safe_join +from pyload.utils import fs_decode, fs_encode, safe_join, safe_path + def chunks(iterable, size): it = iter(iterable) diff --git a/pyload/plugins/account/UptoboxCom.py b/pyload/plugins/account/UptoboxCom.py index 36f541e96..9438888e3 100644 --- a/pyload/plugins/account/UptoboxCom.py +++ b/pyload/plugins/account/UptoboxCom.py @@ -6,7 +6,7 @@ from pyload.plugins.internal.XFSAccount import XFSAccount class UptoboxCom(XFSAccount): __name__ = "UptoboxCom" __type__ = "account" - __version__ = "0.06" + __version__ = "0.07" __description__ = """DDLStorage.com account plugin""" __license__ = "GPLv3" @@ -14,3 +14,4 @@ class UptoboxCom(XFSAccount): HOSTER_DOMAIN = "uptobox.com" + HOSTER_URL = "https://uptobox.com/" diff --git a/pyload/plugins/hoster/EasybytezCom.py b/pyload/plugins/hoster/EasybytezCom.py index b233e52ee..d02cb371b 100644 --- a/pyload/plugins/hoster/EasybytezCom.py +++ b/pyload/plugins/hoster/EasybytezCom.py @@ -6,7 +6,7 @@ from pyload.plugins.internal.XFSHoster import XFSHoster, create_getInfo class EasybytezCom(XFSHoster): __name__ = "EasybytezCom" __type__ = "hoster" - __version__ = "0.22" + __version__ = "0.23" __pattern__ = r'http://(?:www\.)?easybytez\.com/\w{12}' @@ -18,7 +18,6 @@ class EasybytezCom(XFSHoster): HOSTER_DOMAIN = "easybytez.com" - INFO_PATTERN = r'<span class="name">(?P<N>.+)</span><br>\s*<span class="size">(?P<S>[^<]+)</span>' OFFLINE_PATTERN = r'>File not available' LINK_PATTERN = r'(http://(\w+\.(easybytez|easyload|ezbytez|zingload)\.(com|to)|\d+\.\d+\.\d+\.\d+)/files/\d+/\w+/.+?)["\'<]' diff --git a/pyload/plugins/hoster/RarefileNet.py b/pyload/plugins/hoster/RarefileNet.py index fb54f6431..ce54b4dc7 100644 --- a/pyload/plugins/hoster/RarefileNet.py +++ b/pyload/plugins/hoster/RarefileNet.py @@ -3,13 +3,12 @@ import re from pyload.plugins.internal.XFSHoster import XFSHoster, create_getInfo -from pyload.utils import html_unescape class RarefileNet(XFSHoster): __name__ = "RarefileNet" __type__ = "hoster" - __version__ = "0.06" + __version__ = "0.08" __pattern__ = r'http://(?:www\.)?rarefile\.net/\w{12}' @@ -20,19 +19,10 @@ class RarefileNet(XFSHoster): HOSTER_DOMAIN = "rarefile.net" - NAME_PATTERN = r'<td><font color="red">(?P<N>.*?)</font></td>' - SIZE_PATTERN = r'<td>Size : (?P<S>.+?) ' + NAME_PATTERN = r'<font color="red">(?P<N>.+?)<' + SIZE_PATTERN = r'>Size : (?P<S>[\d.,]+) (?P<U>[\w^_]+)' LINK_PATTERN = r'<a href="(?P<link>[^"]+)">(?P=link)</a>' - def handleCaptcha(self, inputs): - captcha_div = re.search(r'<b>Enter code.*?<div.*?>(.*?)</div>', self.html, re.S).group(1) - self.logDebug(captcha_div) - numerals = re.findall('<span.*?padding-left\s*:\s*(\d+).*?>(\d)</span>', html_unescape(captcha_div)) - inputs['code'] = "".join([a[1] for a in sorted(numerals, key=lambda num: int(num[0]))]) - self.logDebug("CAPTCHA", inputs['code'], numerals) - return 3 - - getInfo = create_getInfo(RarefileNet) diff --git a/pyload/plugins/internal/SimpleCrypter.py b/pyload/plugins/internal/SimpleCrypter.py index eb0f8a323..ead5cefba 100644 --- a/pyload/plugins/internal/SimpleCrypter.py +++ b/pyload/plugins/internal/SimpleCrypter.py @@ -5,15 +5,14 @@ import re from urlparse import urlparse from pyload.plugins.internal.Crypter import Crypter -from pyload.plugins.Plugin import Fail -from pyload.plugins.internal.SimpleHoster import _error, _wait, parseFileInfo, replace_patterns, set_cookies -from pyload.utils import fixup, html_unescape +from pyload.plugins.internal.SimpleHoster import SimpleHoster, replace_patterns, set_cookies +from pyload.utils import fixup -class SimpleCrypter(Crypter): +class SimpleCrypter(Crypter, SimpleHoster): __name__ = "SimpleCrypter" __type__ = "crypter" - __version__ = "0.30" + __version__ = "0.31" __pattern__ = r'^unmatchable$' __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] @@ -70,15 +69,15 @@ class SimpleCrypter(Crypter): def prepare(self): + self.info = {} + self.links = [] + if self.LOGIN_ACCOUNT and not self.account: self.fail(_("Required account not found")) if self.LOGIN_PREMIUM and not self.premium: self.fail(_("Required premium account not found")) - self.info = {} - self.links = [] - self.req.setOption("timeout", 120) if isinstance(self.COOKIES, list): @@ -87,13 +86,6 @@ class SimpleCrypter(Crypter): self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) - def preload(self): - self.html = self.load(self.pyfile.url, cookies=bool(self.COOKIES), decode=not self.TEXT_ENCODING) - - if isinstance(self.TEXT_ENCODING, basestring): - self.html = unicode(self.html, self.TEXT_ENCODING) - - def decrypt(self, pyfile): self.prepare() @@ -125,30 +117,19 @@ class SimpleCrypter(Crypter): self.tempOffline() - def checkName(self): + def checkNameSize(self): name = self.info['name'] url = self.info['url'] if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split('/')[-1] - - self.info['folder'] = self.pyfile.name - - self.logDebug("File name: %s" % self.pyfile.name) - + self.pyfile.name = name = self.info['name'] = urlparse(name).path.split('/')[-1] - def checkInfo(self): - self.logDebug(_("File info (previous): %s") % self.info) + folder = self.info['folder'] = name - info = parseFileInfo(self.pyfile.url, self.html or "") - self.info.update(info) - - self.logDebug(_("File info (current): %s") % self.info) - - self.checkName() - self.checkStatus() + self.logDebug("File name: %s" % name, + "File folder: %s" % folder) def getLinks(self): @@ -169,7 +150,3 @@ class SimpleCrypter(Crypter): for p in xrange(2, pages + 1): self.html = self.loadPage(p) self.links += self.getLinks() - - - def error(self, reason="", type="parse"): - return super(SimpleCrypter, self).error(self, reason, type) diff --git a/pyload/plugins/internal/SimpleHoster.py b/pyload/plugins/internal/SimpleHoster.py index 9fd4268dc..0a3d5cf95 100644 --- a/pyload/plugins/internal/SimpleHoster.py +++ b/pyload/plugins/internal/SimpleHoster.py @@ -7,12 +7,14 @@ from urlparse import urlparse from pycurl import FOLLOWLOCATION -from pyload.datatype.PyFile import statusMap +from pyload.datatype.PyFile import statusMap as _statusMap from pyload.network.CookieJar import CookieJar from pyload.network.RequestFactory import getURL from pyload.plugins.internal.Hoster import Hoster from pyload.plugins.Plugin import Fail -from pyload.utils import fixup, html_unescape, parseFileSize +from pyload.utils import fixup, parseFileSize +#@TODO: Adapt and move to PyFile in 0.4.10 +statusMap = {v: k for k, v in _statusMap.iteritems()} def replace_patterns(string, ruleslist): @@ -70,13 +72,15 @@ def parseHtmlForm(attr_str, html, input_names=None): return {}, None #: no matching form found +#: Deprecated def parseFileInfo(plugin, url="", html=""): info = plugin.getInfo(url, html) return info['name'], info['size'], info['status'], info['url'] +#@TODO: Remove in 0.4.10 def create_getInfo(plugin): - return lambda urls: list(plugin.parseInfo(urls)) + return lambda urls: [(info['name'], info['size'], info['status'], info['url']) for info in plugin.parseInfo(urls)] def timestamp(): @@ -101,7 +105,7 @@ def _getDirectLink(self, url): class SimpleHoster(Hoster): __name__ = "SimpleHoster" __type__ = "hoster" - __version__ = "0.56" + __version__ = "0.59" __pattern__ = r'^unmatchable$' @@ -113,7 +117,7 @@ class SimpleHoster(Hoster): """ - Following patterns should be defined by each hoster: + Info patterns should be defined by each hoster: INFO_PATTERN: (optional) Name and Size of the file example: INFO_PATTERN = r'(?P<N>file_name) (?P<S>file_size) (?P<U>size_unit)' @@ -123,17 +127,26 @@ class SimpleHoster(Hoster): SIZE_PATTERN: (optional) Size that will be checked for the file example: SIZE_PATTERN = r'(?P<S>file_size) (?P<U>size_unit)' - OFFLINE_PATTERN: (optional) Checks if the file is yet available online + OFFLINE_PATTERN: (optional) Check if the file is yet available online example: OFFLINE_PATTERN = r'File (deleted|not found)' - TEMP_OFFLINE_PATTERN: (optional) Checks if the file is temporarily offline + TEMP_OFFLINE_PATTERN: (optional) Check if the file is temporarily offline example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' - PREMIUM_ONLY_PATTERN: (optional) Checks if the file can be downloaded only with a premium account + + Error handling patterns are all optional: + + WAIT_PATTERN: (optional) Detect waiting time + example: WAIT_PATTERN = r'' + + PREMIUM_ONLY_PATTERN: (optional) Check if the file can be downloaded only with a premium account example: PREMIUM_ONLY_PATTERN = r'Premium account required' + ERROR_PATTERN: (optional) Detect any error preventing download + example: ERROR_PATTERN = r'' + - Instead overriding handleFree and handlePremium methods now you can define patterns for direct download: + Instead overriding handleFree and handlePremium methods you can define the following patterns for direct download: LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download example: LINK_FREE_PATTERN = r'<div class="link"><a href="(.+?)"' @@ -149,7 +162,8 @@ class SimpleHoster(Hoster): TEXT_ENCODING = False #: Set to True or encoding name if encoding value in http header is not correct COOKIES = True #: or False or list of tuples [(domain, name, value)] FORCE_CHECK_TRAFFIC = False #: Set to True to force checking traffic left for premium account - CHECK_DIRECT_LINK = None #: Set to None self-set to True if self.account else to False + CHECK_DIRECT_LINK = None #: when None self-set to True if self.account else False + MULTI_HOSTER = False #: Set to True to leech other hoster link CONTENT_DISPOSITION = False #: Set to True to replace file name with content-disposition value in http header @@ -157,7 +171,7 @@ class SimpleHoster(Hoster): def parseInfo(cls, urls): for url in urls: url = replace_patterns(url, cls.URL_REPLACEMENTS) - yield cls.getInfo(cls, url) + yield cls.getInfo(url) @classmethod @@ -217,17 +231,31 @@ class SimpleHoster(Hoster): def prepare(self): - self.info = {} - self.link = "" #@TODO: Move to hoster class in 0.4.10 - - if self.CHECK_DIRECT_LINK is None: - self.CHECK_DIRECT_LINK = bool(self.account) + self.info = {} + self.link = "" #@TODO: Move to hoster class in 0.4.10 + self.directDL = False #@TODO: Move to hoster class in 0.4.10 + self.multihost = False #@TODO: Move to hoster class in 0.4.10 self.req.setOption("timeout", 120) if isinstance(self.COOKIES, list): set_cookies(self.req.cj, self.COOKIES) + if (self.MULTI_HOSTER + and self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] + and re.match(self.__pattern__, self.pyfile.url) is None): + + self.logInfo("Multi hoster detected") + + if self.account: + self.multihost = True + return + else: + self.fail(_("Only registered or premium users can use url leech feature")) + + if self.CHECK_DIRECT_LINK is None: + self.directDL = bool(self.account) + self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) @@ -241,24 +269,31 @@ class SimpleHoster(Hoster): def process(self, pyfile): self.prepare() - if self.CHECK_DIRECT_LINK: + if self.multihost: + self.logDebug("Looking for leeched download link...") + self.handleMulti() + + elif self.directDL: self.logDebug("Looking for direct download link...") self.handleDirect() if not self.link: self.preload() - #@TODO: Remove in 0.4.10 if self.html is None: self.fail(_("No html retrieved")) + self.checkErrors() + + premium_only = 'error' in self.info and self.info['error'] == "premium-only" + info = self.getInfo(pyfile.url, self.html) self._updateInfo(info) self.checkNameSize() - premium_only = hasattr(self, 'PREMIUM_ONLY_PATTERN') and re.search(self.PREMIUM_ONLY_PATTERN, self.html) - if not premium_only: #: Usually premium only pages doesn't show any file information + #: Usually premium only pages doesn't show any file information + if not premium_only: self.checkStatus() if self.premium and (not self.FORCE_CHECK_TRAFFIC or self.checkTrafficLeft()): @@ -276,6 +311,30 @@ class SimpleHoster(Hoster): self.download(self.link, disposition=self.CONTENT_DISPOSITION) + def checkErrors(self): + if hasattr(self, 'WAIT_PATTERN'): + m = re.search(self.WAIT_PATTERN, self.html) + if m: + wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec)', m, re.I)]) + self.wait(wait_time, False) + return + + if hasattr(self, 'PREMIUM_ONLY_PATTERN'): + m = re.search(self.PREMIUM_ONLY_PATTERN, self.html) + if m: + self.info['error'] = "premium-only" + return + + if hasattr(self, 'ERROR_PATTERN'): + m = re.search(self.ERROR_PATTERN, self.html) + if m: + e = self.info['error'] = m.group(1) + self.error(e) + + self.info.pop('error', None) + + def checkStatus(self): status = self.info['status'] @@ -286,8 +345,9 @@ class SimpleHoster(Hoster): self.tempOffline() elif status is not 2: - self.error(_("File status: %s") % filter(lambda key, val: val == status, statusMap.iteritems())[0], - _("File info: %s") % self.info) + self.logInfo(_("File status: %s") % statusMap[status], + _("File info: %s") % self.info) + self.error(_("No file info retrieved")) def checkNameSize(self): @@ -298,26 +358,35 @@ class SimpleHoster(Hoster): if name and name != url: self.pyfile.name = name else: - self.pyfile.name = self.info['name'] = urlparse(html_unescape(name)).path.split('/')[-1] + self.pyfile.name = name = self.info['name'] = urlparse(name).path.split('/')[-1] if size > 0: self.pyfile.size = size else: - self.logError(_("File size not found")) + size = "Unknown" - self.logDebug("File name: %s" % self.pyfile.name, "File size: %s" % self.pyfile.size or _("Unknown")) + self.logDebug("File name: %s" % name, + "File size: %s" % size) def checkInfo(self): + self.checkErrors() + self._updateInfo(self.getInfo(self.pyfile.url, self.html or "")) + self.checkNameSize() self.checkStatus() - def _updateInfo(self, info) - self.logDebug(_("File info (previous): %s") % self.info) + #: Deprecated + def getFileInfo(self): + return self.checkInfo() + + + def _updateInfo(self, info): + self.logDebug(_("File info (before update): %s") % self.info) self.info.update(info) - self.logDebug(_("File info (current): %s") % self.info) + self.logDebug(_("File info (after update): %s") % self.info) def handleDirect(self): @@ -333,6 +402,10 @@ class SimpleHoster(Hoster): self.logDebug(_("Direct download link not found")) + def handleMulti(self): #: Multi-hoster handler + pass + + def handleFree(self): if not hasattr(self, 'LINK_FREE_PATTERN'): self.fail(_("Free download not implemented")) diff --git a/pyload/plugins/internal/XFSAccount.py b/pyload/plugins/internal/XFSAccount.py index df28191f1..1e18c09bd 100644 --- a/pyload/plugins/internal/XFSAccount.py +++ b/pyload/plugins/internal/XFSAccount.py @@ -12,7 +12,7 @@ from pyload.plugins.internal.SimpleHoster import parseHtmlForm, set_cookies class XFSAccount(Account): __name__ = "XFSAccount" __type__ = "account" - __version__ = "0.25" + __version__ = "0.26" __description__ = """XFileSharing account plugin""" __license__ = "GPLv3" @@ -92,6 +92,9 @@ class XFSAccount(Account): except Exception, e: self.logError(e) + else: + if premium: + trafficleft = -1 return {'validuntil': validuntil, 'trafficleft': trafficleft, 'premium': premium} diff --git a/pyload/plugins/internal/XFSHoster.py b/pyload/plugins/internal/XFSHoster.py index a9fe2d0ad..3ae9ee05a 100644 --- a/pyload/plugins/internal/XFSHoster.py +++ b/pyload/plugins/internal/XFSHoster.py @@ -16,7 +16,7 @@ from pyload.utils import html_unescape class XFSHoster(SimpleHoster): __name__ = "XFSHoster" __type__ = "hoster" - __version__ = "0.17" + __version__ = "0.22" __pattern__ = r'^unmatchable$' @@ -32,27 +32,30 @@ class XFSHoster(SimpleHoster): URL_REPLACEMENTS = [(r'/(?:embed-)?(\w{12}).*', r'/\1')] #: plus support embedded files - COOKIES = [(HOSTER_DOMAIN, "lang", "english")] + TEXT_ENCODING = False + COOKIES = [(HOSTER_DOMAIN, "lang", "english")] + CHECK_DIRECT_LINK = None + MULTI_HOSTER = False INFO_PATTERN = r'<tr><td align=right><b>Filename:</b></td><td nowrap>(?P<N>[^<]+)</td></tr>\s*.*?<small>\((?P<S>[^<]+)\)</small>' - NAME_PATTERN = r'<input type="hidden" name="fname" value="(?P<N>[^"]+)"' - SIZE_PATTERN = r'You have requested .*\((?P<S>[\d.,]+) ?(?P<U>[\w^_]+)?\)</font>' + NAME_PATTERN = r'(>Filename:</b></td><td nowrap>|name="fname" value="|<span class="name">|<[Tt]itle>.*?Download )(?P<N>.+?)(\s*<|")' + SIZE_PATTERN = r'(>Size:</b></td><td>|>File:.*>|<span class="size">)(?P<S>[\d.,]+)\s*(?P<U>[\w^_]+)' OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' - WAIT_PATTERN = r'<span id="countdown_str">.*?>(\d+)</span>|id="countdown" value=".*?(\d+).*?"' + WAIT_PATTERN = r'<span id="countdown_str">.*?>(\d+)</span>|id="countdown" value=".*?(\d+).*?"' + PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only' + ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter><b>|>Error</td>|>\(ERROR:)(?:\s*<.+?>\s*)*(.+?)(?:["\']|<|\))' OVR_LINK_PATTERN = r'<h2>Download Link</h2>\s*<textarea[^>]*>([^<]+)' LINK_PATTERN = None #: final download url pattern - CAPTCHA_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)' + CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' CAPTCHA_DIV_PATTERN = r'>Enter code.*?<div.*?>(.+?)</div>' RECAPTCHA_PATTERN = None SOLVEMEDIA_PATTERN = None - ERROR_PATTERN = r'(?:class=["\']err["\'][^>]*>|<[Cc]enter><b>)(.+?)(?:["\']|</)|>\(ERROR:(.+?)\)' - def setup(self): self.chunkLimit = 1 @@ -71,23 +74,14 @@ class XFSHoster(SimpleHoster): pattern = r'(https?://(www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]' self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.') - if self.CHECK_DIRECT_LINK is None: - self.CHECK_DIRECT_LINK = bool(self.premium) - - self.captcha = None - self.errmsg = None + self.captcha = None + self.errmsg = None self.passwords = self.getPassword().splitlines() - if (self.__pattern__ != self.core.pluginManager.hosterPlugins[self.__name__]['pattern'] - and re.match(self.__pattern__, self.pyfile.url) is None): - self.logInfo(_("Multi hoster detected")) - if self.premium: - self.logDebug(_("Looking for download link...")) - self.handleOverriden() - else: - self.fail(_("Only premium users can use url leech feature")) + super(XFSHoster, self).prepare() - return super(XFSHoster, self).prepare() + if self.CHECK_DIRECT_LINK is None: + self.directDL = bool(self.premium) def handleFree(self): @@ -128,26 +122,26 @@ class XFSHoster(SimpleHoster): self.req.http.c.setopt(FOLLOWLOCATION, 0) self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True) - self.header = self.req.http.header self.req.http.c.setopt(FOLLOWLOCATION, 1) - m = re.search(r'Location\s*:\s*(.+)', self.header, re.I) - if m: + m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) + if m and not "op=" in m.group(1): break m = re.search(self.LINK_PATTERN, self.html, re.S) if m: break else: - return + self.logError(data['op'] if 'op' in data else _("UNKNOWN")) + return "" self.errmsg = None return m.group(1) - def handleOverriden(self): + def handleMulti(self): #only tested with easybytez.com self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) @@ -197,27 +191,36 @@ class XFSHoster(SimpleHoster): self.error(_("OVR_LINK_PATTERN not found")) header = self.load(m.group(1), just_header=True, decode=True) + if 'location' in header: #: Direct download link - self.download(header['location'], ref=True, cookies=True, disposition=True) + self.link = header['location'] else: self.fail(_("Download link not found")) def checkErrors(self): + m = re.search(self.PREMIUM_ONLY_PATTERN, self.html) + if m: + self.info['error'] = "premium-only" + return + m = re.search(self.ERROR_PATTERN, self.html) if m is None: self.errmsg = None else: - self.errmsg = m.group(1) + self.errmsg = m.group(1).strip() self.logWarning(re.sub(r"<.*?>", " ", self.errmsg)) if 'wait' in self.errmsg: - wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in - re.findall(r'(\d+)\s*(hour|minute|second)', self.errmsg)]) + wait_time = sum([int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec)', self.errmsg, re.I)]) self.wait(wait_time, True) + elif 'country' in self.errmsg: + self.fail(_("Downloads are disabled for your country")) + elif 'captcha' in self.errmsg: self.invalidCaptcha() @@ -247,62 +250,57 @@ class XFSHoster(SimpleHoster): else: self.retry(wait_time=60, reason=self.errmsg) + if self.errmsg: + self.info['error'] = self.errmsg + else: + self.info.pop('error', None) + return self.errmsg def getPostParameters(self): - for _i in xrange(3): - if hasattr(self, "FORM_PATTERN"): - action, inputs = self.parseHtmlForm(self.FORM_PATTERN) - else: - action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) + if hasattr(self, "FORM_PATTERN"): + action, inputs = self.parseHtmlForm(self.FORM_PATTERN) + else: + action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")}) + if not inputs: + action, inputs = self.parseHtmlForm('F1') if not inputs: - action, inputs = self.parseHtmlForm('F1') - if not inputs: - if self.errmsg: - self.retry(reason=self.errmsg) - else: - self.error(_("TEXTAREA F1 not found")) - - self.logDebug(inputs) - - if 'op' in inputs and inputs['op'] in ("download1", "download2", "download3"): - if "password" in inputs: - if self.passwords: - inputs['password'] = self.passwords.pop(0) - else: - self.fail(_("No or invalid passport")) - - if not self.premium: - m = re.search(self.WAIT_PATTERN, self.html) - if m: - wait_time = int(m.group(1)) - self.setWait(wait_time, False) - else: - wait_time = 0 - - self.captcha = self.handleCaptcha(inputs) - - if wait_time: - self.wait() - - return inputs - else: - inputs['referer'] = self.pyfile.url + if self.errmsg: + self.retry(reason=self.errmsg) + else: + self.error(_("TEXTAREA F1 not found")) - if self.premium: - inputs['method_premium'] = "Premium Download" - if 'method_free' in inputs: - del inputs['method_free'] + self.logDebug(inputs) + + if 'op' in inputs: + if "password" in inputs: + if self.passwords: + inputs['password'] = self.passwords.pop(0) else: - inputs['method_free'] = "Free Download" - if 'method_premium' in inputs: - del inputs['method_premium'] + self.fail(_("Missing password")) + + if not self.premium: + m = re.search(self.WAIT_PATTERN, self.html) + if m: + wait_time = int(m.group(1)) + self.setWait(wait_time, False) + + self.captcha = self.handleCaptcha(inputs) - self.html = self.load(self.pyfile.url, post=inputs, ref=True) + self.wait() else: - self.error(_("FORM: %s") % (inputs['op'] if 'op' in inputs else _("UNKNOWN"))) + inputs['referer'] = self.pyfile.url + + if self.premium: + inputs['method_premium'] = "Premium Download" + inputs.pop('method_free', None) + else: + inputs['method_free'] = "Free Download" + inputs.pop('method_premium', None) + + return inputs def handleCaptcha(self, inputs): |