From 8e7d14bae4d3c836f029a1235eb227380acc3f75 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Mon, 16 Feb 2015 21:59:10 +0100 Subject: Fix plugins to work on 0.4.10 --- pyload/plugin/internal/BasePlugin.py | 92 ++++ pyload/plugin/internal/DeadCrypter.py | 27 ++ pyload/plugin/internal/DeadHoster.py | 27 ++ pyload/plugin/internal/MultiHook.py | 308 +++++++++++++ pyload/plugin/internal/MultiHoster.py | 85 ++++ pyload/plugin/internal/SimpleCrypter.py | 157 +++++++ pyload/plugin/internal/SimpleDereferer.py | 98 +++++ pyload/plugin/internal/SimpleHoster.py | 701 ++++++++++++++++++++++++++++++ pyload/plugin/internal/UpdateManager.py | 306 +++++++++++++ pyload/plugin/internal/XFSAccount.py | 174 ++++++++ pyload/plugin/internal/XFSCrypter.py | 45 ++ pyload/plugin/internal/XFSHoster.py | 326 ++++++++++++++ pyload/plugin/internal/__init__.py | 1 + 13 files changed, 2347 insertions(+) create mode 100644 pyload/plugin/internal/BasePlugin.py create mode 100644 pyload/plugin/internal/DeadCrypter.py create mode 100644 pyload/plugin/internal/DeadHoster.py create mode 100644 pyload/plugin/internal/MultiHook.py create mode 100644 pyload/plugin/internal/MultiHoster.py create mode 100644 pyload/plugin/internal/SimpleCrypter.py create mode 100644 pyload/plugin/internal/SimpleDereferer.py create mode 100644 pyload/plugin/internal/SimpleHoster.py create mode 100644 pyload/plugin/internal/UpdateManager.py create mode 100644 pyload/plugin/internal/XFSAccount.py create mode 100644 pyload/plugin/internal/XFSCrypter.py create mode 100644 pyload/plugin/internal/XFSHoster.py create mode 100644 pyload/plugin/internal/__init__.py (limited to 'pyload/plugin/internal') diff --git a/pyload/plugin/internal/BasePlugin.py b/pyload/plugin/internal/BasePlugin.py new file mode 100644 index 000000000..103e0d5cb --- /dev/null +++ b/pyload/plugin/internal/BasePlugin.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +import re + +from urllib import unquote +from urlparse import urljoin, urlparse + +from pyload.network.HTTPRequest import BadHeader +from pyload.plugin.internal.SimpleHoster import fileUrl +from pyload.plugin.Hoster import Hoster + + +class BasePlugin(Hoster): + __name__ = "BasePlugin" + __type__ = "hoster" + __version__ = "0.34" + + __pattern__ = r'^unmatchable$' + + __description__ = """Base plugin when any other didnt fit""" + __license__ = "GPLv3" + __authors__ = [("RaNaN", "RaNaN@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + @classmethod + def getInfo(cls, url="", html=""): #@TODO: Move to hoster class in 0.4.10 + url = unquote(url) + url_p = urlparse(url) + return {'name' : (url_p.path.split('/')[-1] + or url_p.query.split('=', 1)[::-1][0].split('&', 1)[0] + or url_p.netloc.split('.', 1)[0]), + 'size' : 0, + 'status': 3 if url else 8, + 'url' : url} + + + def setup(self): + self.chunkLimit = -1 + self.multiDL = True + self.resumeDownload = True + + + def process(self, pyfile): + """main function""" + + pyfile.name = self.getInfo(pyfile.url)['name'] + + if not pyfile.url.startswith("http"): + self.fail(_("No plugin matched")) + + for _i in xrange(5): + try: + link = fileUrl(self, unquote(pyfile.url)) + + if link: + self.download(link, ref=False, disposition=True) + else: + self.fail(_("File not found")) + + except BadHeader, e: + if e.code is 404: + self.offline() + + elif e.code in (401, 403): + self.logDebug("Auth required", "Received HTTP status code: %d" % e.code) + + account = self.core.accountManager.getAccountPlugin('Http') + servers = [x['login'] for x in account.getAllAccounts()] + server = urlparse(pyfile.url).netloc + + if server in servers: + self.logDebug("Logging on to %s" % server) + self.req.addAuth(account.getAccountData(server)['password']) + else: + pwd = self.getPassword() + if ':' in pwd: + self.req.addAuth(pwd) + else: + self.fail(_("Authorization required")) + else: + self.fail(e) + else: + break + else: + self.fail(_("No file downloaded")) #@TODO: Move to hoster class in 0.4.10 + + check = self.checkDownload({'empty file': re.compile(r'\A\Z'), + 'html file' : re.compile(r'\A\s*)?\d{3}(\Z|\s+)')}) + if check: + self.fail(check.capitalize()) diff --git a/pyload/plugin/internal/DeadCrypter.py b/pyload/plugin/internal/DeadCrypter.py new file mode 100644 index 000000000..ddeb0431d --- /dev/null +++ b/pyload/plugin/internal/DeadCrypter.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Crypter import Crypter as _Crypter + + +class DeadCrypter(_Crypter): + __name__ = "DeadCrypter" + __type__ = "crypter" + __version__ = "0.04" + + __pattern__ = r'^unmatchable$' + + __description__ = """Crypter is no longer available""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it")] + + + @classmethod + def apiInfo(cls, url="", get={}, post={}): + api = super(DeadCrypter, self).apiInfo(url, get, post) + api['status'] = 1 + return api + + + def setup(self): + self.pyfile.error = "Crypter is no longer available" + self.offline() #@TODO: self.offline("Crypter is no longer available") diff --git a/pyload/plugin/internal/DeadHoster.py b/pyload/plugin/internal/DeadHoster.py new file mode 100644 index 000000000..1596943ae --- /dev/null +++ b/pyload/plugin/internal/DeadHoster.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.Hoster import Hoster as _Hoster + + +class DeadHoster(_Hoster): + __name__ = "DeadHoster" + __type__ = "hoster" + __version__ = "0.14" + + __pattern__ = r'^unmatchable$' + + __description__ = """Hoster is no longer available""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz")] + + + @classmethod + def apiInfo(cls, url="", get={}, post={}): + api = super(DeadHoster, self).apiInfo(url, get, post) + api['status'] = 1 + return api + + + def setup(self): + self.pyfile.error = "Hoster is no longer available" + self.offline() #@TODO: self.offline("Hoster is no longer available") diff --git a/pyload/plugin/internal/MultiHook.py b/pyload/plugin/internal/MultiHook.py new file mode 100644 index 000000000..2beccfcc5 --- /dev/null +++ b/pyload/plugin/internal/MultiHook.py @@ -0,0 +1,308 @@ +# -*- coding: utf-8 -*- + +import re + +from time import sleep + +from pyload.plugin.Hook import Hook +from pyload.utils import decode, remove_chars + + +class MultiHook(Hook): + __name__ = "MultiHook" + __type__ = "hook" + __version__ = "0.37" + + __config__ = [("pluginmode" , "all;listed;unlisted", "Use for plugins" , "all"), + ("pluginlist" , "str" , "Plugin list (comma separated)" , "" ), + ("revertfailed" , "bool" , "Revert to standard download if fails", True ), + ("retry" , "int" , "Number of retries before revert" , 10 ), + ("retryinterval" , "int" , "Retry interval in minutes" , 1 ), + ("reload" , "bool" , "Reload plugin list" , True ), + ("reloadinterval", "int" , "Reload interval in hours" , 12 )] + + __description__ = """Hook plugin for multi hoster/crypter""" + __license__ = "GPLv3" + __authors__ = [("pyLoad Team", "admin@pyload.org"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + MIN_INTERVAL = 1 * 60 * 60 + + DOMAIN_REPLACEMENTS = [(r'180upload\.com' , "hundredeightyupload.com"), + (r'1fichier\.com' , "onefichier.com" ), + (r'2shared\.com' , "twoshared.com" ), + (r'4shared\.com' , "fourshared.com" ), + (r'bayfiles\.net' , "bayfiles.com" ), + (r'cloudnator\.com' , "shragle.com" ), + (r'dfiles\.eu' , "depositfiles.com" ), + (r'easy-share\.com' , "crocko.com" ), + (r'freakshare\.net' , "freakshare.com" ), + (r'hellshare\.com' , "hellshare.cz" ), + (r'ifile\.it' , "filecloud.io" ), + (r'nowdownload\.\w+', "nowdownload.sx" ), + (r'nowvideo\.\w+' , "nowvideo.sx" ), + (r'putlocker\.com' , "firedrive.com" ), + (r'share-?rapid\.cz', "multishare.cz" ), + (r'ul\.to' , "uploaded.to" ), + (r'uploaded\.net' , "uploaded.to" ), + (r'uploadhero\.co' , "uploadhero.com" ), + (r'zshares\.net' , "zshare.net" ), + (r'(\d+.+)' , "X\1" )] + + + def setup(self): + self.plugins = [] + self.supported = [] + self.new_supported = [] + + self.account = None + self.pluginclass = None + self.pluginmodule = None + self.pluginname = None + self.plugintype = None + + self._initPlugin() + + + def _initPlugin(self): + plugin, type = self.core.pluginManager.findPlugin(self.__name__) + + if not plugin: + self.logWarning("Hook plugin will be deactivated due missing plugin reference") + self.setConfig('activated', False) + else: + self.pluginname = self.__name__ + self.plugintype = type + self.pluginmodule = self.core.pluginManager.loadModule(type, self.__name__) + self.pluginclass = getattr(self.pluginmodule, self.__name__) + + + def _loadAccount(self): + self.account = self.core.accountManager.getAccountPlugin(self.pluginname) + + if self.account and not self.account.canUse(): + self.account = None + + if not self.account and hasattr(self.pluginclass, "LOGIN_ACCOUNT") and self.pluginclass.LOGIN_ACCOUNT: + self.logWarning("Hook plugin will be deactivated due missing account reference") + self.setConfig('activated', False) + + + def activate(self): + self._loadAccount() + + + def getURL(self, *args, **kwargs): #@TODO: Remove in 0.4.10 + """ see HTTPRequest for argument list """ + h = pyreq.getHTTPRequest(timeout=120) + try: + if not 'decode' in kwargs: + kwargs['decode'] = True + rep = h.load(*args, **kwargs) + finally: + h.close() + + return rep + + + def getConfig(self, option, default=''): + """getConfig with default value - sublass may not implements all config options""" + try: + return self.getConf(option) + + except KeyError: + return default + + + def pluginsCached(self): + if self.plugins: + return self.plugins + + for _i in xrange(3): + try: + pluginset = self._pluginSet(self.getHosters() if self.plugintype == "hoster" else self.getCrypters()) + + except Exception, e: + self.logError(e, "Waiting 1 minute and retry") + sleep(60) + + else: + break + else: + return list() + + try: + configmode = self.getConfig("pluginmode", 'all') + if configmode in ("listed", "unlisted"): + pluginlist = self.getConfig("pluginlist", '').replace('|', ',').replace(';', ',').split(',') + configset = self._pluginSet(pluginlist) + + if configmode == "listed": + pluginset &= configset + else: + pluginset -= configset + + except Exception, e: + self.logError(e) + + self.plugins = list(pluginset) + + return self.plugins + + + def _pluginSet(self, plugins): + plugins = set((decode(x).strip().lower() for x in plugins if '.' in x)) + + for rf, rt in self.DOMAIN_REPLACEMENTS: + regex = re.compile(rf) + for p in filter(lambda x: regex.match(x), plugins): + plugins.remove(p) + plugins.add(re.sub(rf, rt, p)) + + plugins.discard('') + + return plugins + + + def getHosters(self): + """Load list of supported hoster + + :return: List of domain names + """ + raise NotImplementedError + + + def getCrypters(self): + """Load list of supported crypters + + :return: List of domain names + """ + raise NotImplementedError + + + def periodical(self): + """reload plugin list periodically""" + self.logInfo(_("Reloading supported %s list") % self.plugintype) + + old_supported = self.supported + + self.supported = [] + self.new_supported = [] + self.plugins = [] + + self.overridePlugins() + + old_supported = [plugin for plugin in old_supported if plugin not in self.supported] + + if old_supported: + self.logDebug("Unload: %s" % ", ".join(old_supported)) + for plugin in old_supported: + self.unloadPlugin(plugin) + + if self.getConfig("reload", True): + self.interval = max(self.getConfig("reloadinterval", 12) * 60 * 60, self.MIN_INTERVAL) + else: + self.core.scheduler.removeJob(self.cb) + self.cb = None + + + def overridePlugins(self): + excludedList = [] + + if self.plugintype == "hoster": + pluginMap = dict((name.lower(), name) for name in self.core.pluginManager.hosterPlugins.iterkeys()) + accountList = [account.type.lower() for account in self.core.api.getAccounts(False) if account.valid and account.premium] + else: + pluginMap = {} + accountList = [name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in self.core.pluginManager.crypterPlugins.iterkeys()] + + for plugin in self.pluginsCached(): + name = remove_chars(plugin, "-.") + + if name in accountList: + excludedList.append(plugin) + else: + if name in pluginMap: + self.supported.append(pluginMap[name]) + else: + self.new_supported.append(plugin) + + if not self.supported and not self.new_supported: + self.logError(_("No %s loaded") % self.plugintype) + return + + # inject plugin plugin + self.logDebug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) + + for plugin in self.supported: + hdict = self.core.pluginManager.plugins[self.plugintype][plugin] + hdict['new_module'] = self.pluginmodule + hdict['new_name'] = self.pluginname + + if excludedList: + self.logInfo(_("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excludedList)))) + + if self.new_supported: + plugins = sorted(self.new_supported) + + self.logDebug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) + + # create new regexp + regexp = r'.*(?P%s).*' % "|".join([x.replace(".", "\.") for x in plugins]) + if hasattr(self.pluginclass, "__pattern__") and isinstance(self.pluginclass.__pattern__, basestring) and '://' in self.pluginclass.__pattern__: + regexp = r'%s|%s' % (self.pluginclass.__pattern__, regexp) + + self.logDebug("Regexp: %s" % regexp) + + hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] + hdict['pattern'] = regexp + hdict['re'] = re.compile(regexp) + + + def unloadPlugin(self, plugin): + hdict = self.core.pluginManager.plugins[self.plugintype][plugin] + if "module" in hdict: + del hdict['module'] + + if "new_module" in hdict: + del hdict['new_module'] + del hdict['new_name'] + + + def deactivate(self): + """Remove override for all plugins. Scheduler job is removed by hookmanager""" + for plugin in self.supported: + self.unloadPlugin(plugin) + + # reset pattern + hdict = self.core.pluginManager.plugins[self.plugintype][self.pluginname] + + hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') + hdict['re'] = re.compile(hdict['pattern']) + + + def downloadFailed(self, pyfile): + """remove plugin override if download fails but not if file is offline/temp.offline""" + if pyfile.status != 8 or not self.getConfig("revertfailed", True): + return + + hdict = self.core.pluginManager.plugins[self.plugintype][pyfile.pluginname] + if "new_name" in hdict and hdict['new_name'] == self.pluginname: + if pyfile.error == "MultiHook": + self.logDebug("Unload MultiHook", pyfile.pluginname, hdict) + self.unloadPlugin(pyfile.pluginname) + pyfile.setStatus("queued") + pyfile.sync() + else: + retries = max(self.getConfig("retry", 10), 0) + wait_time = max(self.getConfig("retryinterval", 1), 0) + + if 0 < retries > pyfile.plugin.retries: + self.logInfo(_("Retrying: %s") % pyfile.name) + pyfile.setCustomStatus("MultiHook", "queued") + pyfile.sync() + + pyfile.plugin.retries += 1 + pyfile.plugin.setWait(wait_time) + pyfile.plugin.wait() diff --git a/pyload/plugin/internal/MultiHoster.py b/pyload/plugin/internal/MultiHoster.py new file mode 100644 index 000000000..ed425ffaa --- /dev/null +++ b/pyload/plugin/internal/MultiHoster.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- + +import re + +from pyload.plugin.internal.SimpleHoster import SimpleHoster, replace_patterns, set_cookies + + +class MultiHoster(SimpleHoster): + __name__ = "MultiHoster" + __type__ = "hoster" + __version__ = "0.37" + + __pattern__ = r'^unmatchable$' + + __description__ = """Multi hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + LOGIN_ACCOUNT = True + + + def setup(self): + self.chunkLimit = 1 + self.multiDL = bool(self.account) + self.resumeDownload = self.premium + + + def prepare(self): + self.info = {} + self.html = "" + self.link = "" #@TODO: Move to hoster class in 0.4.10 + self.directDL = False #@TODO: Move to hoster class in 0.4.10 + + if self.LOGIN_ACCOUNT and not self.account: + self.fail(_("Required account not found")) + + self.req.setOption("timeout", 120) + + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) + + if self.DIRECT_LINK is None: + self.directDL = self.__pattern__ != r'^unmatchable$' and re.match(self.__pattern__, self.pyfile.url) + else: + self.directDL = self.DIRECT_LINK + + self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) + + + def process(self, pyfile): + self.prepare() + + if self.directDL: + self.checkInfo() + self.logDebug("Looking for direct download link...") + self.handleDirect(pyfile) + + if not self.link and not self.lastDownload: + self.preload() + + self.checkErrors() + self.checkStatus(getinfo=False) + + if self.premium and (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as premium download") + self.handlePremium(pyfile) + + elif not self.LOGIN_ACCOUNT or (not self.CHECK_TRAFFIC or self.checkTrafficLeft()): + self.logDebug("Handled as free download") + self.handleFree(pyfile) + + self.downloadLink(self.link, True) + self.checkFile() + + + def handlePremium(self, pyfile): + return self.handleFree(pyfile) + + + def handleFree(self, pyfile): + if self.premium: + raise NotImplementedError + else: + self.fail(_("Required premium account not found")) diff --git a/pyload/plugin/internal/SimpleCrypter.py b/pyload/plugin/internal/SimpleCrypter.py new file mode 100644 index 000000000..e4b8874f3 --- /dev/null +++ b/pyload/plugin/internal/SimpleCrypter.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- + +import re + +from urlparse import urljoin, urlparse + +from pyload.plugin.Crypter import Crypter +from pyload.plugin.internal.SimpleHoster import SimpleHoster, replace_patterns, set_cookies +from pyload.utils import fixup + + +class SimpleCrypter(Crypter, SimpleHoster): + __name__ = "SimpleCrypter" + __type__ = "crypter" + __version__ = "0.43" + + __pattern__ = r'^unmatchable$' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), #: Overrides core.config['general']['folder_per_package'] + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Simple decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("stickell", "l.stickell@yahoo.it"), + ("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + """ + Following patterns should be defined by each crypter: + + LINK_PATTERN: Download link or regex to catch links in group(1) + example: LINK_PATTERN = r'
Files of: (?P[^<]+) folder' + + OFFLINE_PATTERN: (optional) Checks if the page is unreachable + example: OFFLINE_PATTERN = r'File (deleted|not found)' + + TEMP_OFFLINE_PATTERN: (optional) Checks if the page is temporarily unreachable + example: TEMP_OFFLINE_PATTERN = r'Server maintainance' + + + You can override the getLinks method if you need a more sophisticated way to extract the links. + + + If the links are splitted on multiple pages you can define the PAGES_PATTERN regex: + + PAGES_PATTERN: (optional) group(1) should be the number of overall pages containing the links + example: PAGES_PATTERN = r'Pages: (\d+)' + + and its loadPage method: + + + def loadPage(self, page_n): + return the html of the page number page_n + """ + + LINK_PATTERN = None + + NAME_REPLACEMENTS = [("&#?\w+;", fixup)] + URL_REPLACEMENTS = [] + + TEXT_ENCODING = False #: Set to True or encoding name if encoding in http header is not correct + COOKIES = True #: or False or list of tuples [(domain, name, value)] + + LOGIN_ACCOUNT = False + LOGIN_PREMIUM = False + + + def prepare(self): + self.pyfile.error = "" #@TODO: Remove in 0.4.10 + + self.info = {} + self.html = "" + self.links = [] #@TODO: Move to hoster class in 0.4.10 + + if self.LOGIN_PREMIUM and not self.premium: + self.fail(_("Required premium account not found")) + + if self.LOGIN_ACCOUNT and not self.account: + self.fail(_("Required account not found")) + + self.req.setOption("timeout", 120) + + if isinstance(self.COOKIES, list): + set_cookies(self.req.cj, self.COOKIES) + + self.pyfile.url = replace_patterns(self.pyfile.url, self.URL_REPLACEMENTS) + + + def decrypt(self, pyfile): + self.prepare() + + self.preload() + self.checkInfo() + + self.links = self.getLinks() + + if hasattr(self, 'PAGES_PATTERN') and hasattr(self, 'loadPage'): + self.handlePages(pyfile) + + self.logDebug("Package has %d links" % len(self.links)) + + if self.links: + self.packages = [(self.info['name'], self.links, self.info['folder'])] + + elif not self.urls and not self.packages: #@TODO: Remove in 0.4.10 + self.fail(_("No link grabbed")) + + + def checkNameSize(self, getinfo=True): + if not self.info or getinfo: + self.logDebug("File info (BEFORE): %s" % self.info) + self.info.update(self.getInfo(self.pyfile.url, self.html)) + self.logDebug("File info (AFTER): %s" % self.info) + + try: + url = self.info['url'].strip() + name = self.info['name'].strip() + if name and name != url: + self.pyfile.name = name + + except Exception: + pass + + try: + folder = self.info['folder'] = self.pyfile.name + + except Exception: + pass + + self.logDebug("File name: %s" % self.pyfile.name, + "File folder: %s" % self.pyfile.name) + + + def getLinks(self): + """ + Returns the links extracted from self.html + You should override this only if it's impossible to extract links using only the LINK_PATTERN. + """ + url_p = urlparse(self.pyfile.url) + baseurl = "%s://%s" % (url_p.scheme, url_p.netloc) + + return [urljoin(baseurl, link) if not urlparse(link).scheme else link \ + for link in re.findall(self.LINK_PATTERN, self.html)] + + + def handlePages(self, pyfile): + try: + pages = int(re.search(self.PAGES_PATTERN, self.html).group(1)) + except Exception: + pages = 1 + + for p in xrange(2, pages + 1): + self.html = self.loadPage(p) + self.links += self.getLinks() diff --git a/pyload/plugin/internal/SimpleDereferer.py b/pyload/plugin/internal/SimpleDereferer.py new file mode 100644 index 000000000..6d323b4b0 --- /dev/null +++ b/pyload/plugin/internal/SimpleDereferer.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- + +import re + +from urllib import unquote + +from pyload.plugin.Crypter import Crypter +from pyload.plugin.internal.SimpleHoster import fileUrl, set_cookies + + +class SimpleDereferer(Crypter): + __name__ = "SimpleDereferer" + __type__ = "crypter" + __version__ = "0.07" + + __pattern__ = r'^unmatchable$' + __config__ = [("use_subfolder", "bool", "Save package to subfolder", True), + ("subfolder_per_package", "bool", "Create a subfolder for each package", True)] + + __description__ = """Simple dereferer plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + """ + Following patterns should be defined by each crypter: + + LINK_PATTERN: Regex to catch the redirect url in group(1) + example: LINK_PATTERN = r'
\s\"'][^>\s]*)\1" % attr_name, tag, re.I) + return m.group(2) if m else None + + +def parseHtmlForm(attr_str, html, input_names={}): + for form in re.finditer(r"(?P]*%s[^>]*>)(?P.*?)]*>" % attr_str, + html, re.S | re.I): + inputs = {} + action = parseHtmlTagAttrValue("action", form.group('TAG')) + + for inputtag in re.finditer(r'(<(input|textarea)[^>]*>)([^<]*(?=file_name) (?Pfile_size) (?Psize_unit)' + or + NAME_PATTERN: (optional) Name that will be set for the file + example: NAME_PATTERN = r'(?Pfile_name)' + SIZE_PATTERN: (optional) Size that will be checked for the file + example: SIZE_PATTERN = r'(?Pfile_size) (?Psize_unit)' + + HASHSUM_PATTERN: (optional) Hash code and type of the file + example: HASHSUM_PATTERN = r'(?Phash_code) (?PMD5)' + + OFFLINE_PATTERN: (optional) Check if the page is unreachable + example: OFFLINE_PATTERN = r'File (deleted|not found)' + + TEMP_OFFLINE_PATTERN: (optional) Check if the page is temporarily unreachable + example: TEMP_OFFLINE_PATTERN = r'Server (maintenance|maintainance)' + + + Error handling patterns are all optional: + + WAIT_PATTERN: (optional) Detect waiting time + example: WAIT_PATTERN = r'' + + PREMIUM_ONLY_PATTERN: (optional) Check if the file can be downloaded only with a premium account + example: PREMIUM_ONLY_PATTERN = r'Premium account required' + + ERROR_PATTERN: (optional) Detect any error preventing download + example: ERROR_PATTERN = r'' + + + Instead overriding handleFree and handlePremium methods you can define the following patterns for direct download: + + LINK_FREE_PATTERN: (optional) group(1) should be the direct link for free download + example: LINK_FREE_PATTERN = r'
)?\d{3}(\Z|\s+)')} + + if hasattr(self, 'ERROR_PATTERN'): + rules['error'] = re.compile(self.ERROR_PATTERN) + + check = self.checkDownload(rules) + if check: #@TODO: Move to hoster in 0.4.10 + errmsg = check.strip().capitalize() + if self.lastCheck: + errmsg += " | " + self.lastCheck.group(0).strip() + + self.lastDownload = "" + self.retry(10, 60, errmsg) + + + def checkErrors(self): + if not self.html: + self.logWarning(_("No html code to check")) + return + + if hasattr(self, 'PREMIUM_ONLY_PATTERN') and not self.premium and re.search(self.PREMIUM_ONLY_PATTERN, self.html): + self.fail(_("Link require a premium account to be handled")) + + elif hasattr(self, 'ERROR_PATTERN'): + m = re.search(self.ERROR_PATTERN, self.html) + if m: + errmsg = self.info['error'] = m.group(1) + self.error(errmsg) + + elif hasattr(self, 'WAIT_PATTERN'): + m = re.search(self.WAIT_PATTERN, self.html) + if m: + wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec)', m.group(0), re.I)) + self.wait(wait_time, wait_time > 300) + return + + self.info.pop('error', None) + + + def checkStatus(self, getinfo=True): + if not self.info or getinfo: + self.logDebug("Update file info...") + self.logDebug("Previous file info: %s" % self.info) + self.info.update(self.getInfo(self.pyfile.url, self.html)) + self.logDebug("Current file info: %s" % self.info) + + try: + status = self.info['status'] + + if status is 1: + self.offline() + + elif status is 6: + self.tempOffline() + + elif status is 8: + self.fail() + + finally: + self.logDebug("File status: %s" % statusMap[status]) + + + def checkNameSize(self, getinfo=True): + if not self.info or getinfo: + self.logDebug("Update file info...") + self.logDebug("Previous file info: %s" % self.info) + self.info.update(self.getInfo(self.pyfile.url, self.html)) + self.logDebug("Current file info: %s" % self.info) + + try: + url = self.info['url'].strip() + name = self.info['name'].strip() + if name and name != url: + self.pyfile.name = name + + except Exception: + pass + + try: + size = self.info['size'] + if size > 0: + self.pyfile.size = size + + except Exception: + pass + + self.logDebug("File name: %s" % self.pyfile.name, + "File size: %s byte" % self.pyfile.size if self.pyfile.size > 0 else "File size: Unknown") + + + def checkInfo(self): + self.checkNameSize() + + if self.html: + self.checkErrors() + self.checkNameSize() + + self.checkStatus(getinfo=False) + + + #: Deprecated + def getFileInfo(self): + self.info = {} + self.checkInfo() + return self.info + + + def handleDirect(self, pyfile): + link = self.directLink(pyfile.url, self.resumeDownload) + + if link: + self.logInfo(_("Direct download link detected")) + + self.link = link + else: + self.logDebug("Direct download link not found") + + + def handleMulti(self, pyfile): #: Multi-hoster handler + pass + + + def handleFree(self, pyfile): + if not hasattr(self, 'LINK_FREE_PATTERN'): + self.logError(_("Free download not implemented")) + + m = re.search(self.LINK_FREE_PATTERN, self.html) + if m is None: + self.error(_("Free download link not found")) + else: + self.link = m.group(1) + + + def handlePremium(self, pyfile): + if not hasattr(self, 'LINK_PREMIUM_PATTERN'): + self.logError(_("Premium download not implemented")) + self.logDebug("Handled as free download") + self.handleFree(pyfile) + + m = re.search(self.LINK_PREMIUM_PATTERN, self.html) + if m is None: + self.error(_("Premium download link not found")) + else: + self.link = m.group(1) + + + def longWait(self, wait_time=None, max_tries=3): + if wait_time and isinstance(wait_time, (int, long, float)): + time_str = "%dh %dm" % divmod(wait_time / 60, 60) + else: + wait_time = 900 + time_str = _("(unknown time)") + max_tries = 100 + + self.logInfo(_("Download limit reached, reconnect or wait %s") % time_str) + + self.wait(wait_time, True) + self.retry(max_tries=max_tries, reason=_("Download limit reached")) + + + def parseHtmlForm(self, attr_str="", input_names={}): + return parseHtmlForm(attr_str, self.html, input_names) + + + def checkTrafficLeft(self): + if not self.account: + return True + + traffic = self.account.getAccountInfo(self.user, True)['trafficleft'] + + if traffic is None: + return False + elif traffic == -1: + return True + else: + size = self.pyfile.size / 1024 + self.logInfo(_("Filesize: %i KiB, Traffic left for user %s: %i KiB") % (size, self.user, traffic)) + return size <= traffic + + + #@TODO: Remove in 0.4.10 + def wait(self, seconds=0, reconnect=None): + return _wait(self, seconds, reconnect) + + + def error(self, reason="", type="parse"): + return _error(self, reason, type) diff --git a/pyload/plugin/internal/UpdateManager.py b/pyload/plugin/internal/UpdateManager.py new file mode 100644 index 000000000..ff692b4ae --- /dev/null +++ b/pyload/plugin/internal/UpdateManager.py @@ -0,0 +1,306 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import re +import sys + +from operator import itemgetter +from os import path, remove, stat + +from pyload.network.RequestFactory import getURL +from pyload.plugin.Addon import Expose, Addon, threaded +from pyload.utils import safe_join + + +class UpdateManager(Addon): + __name__ = "UpdateManager" + __type__ = "addon" + __version__ = "0.43" + + __config__ = [("activated" , "bool" , "Activated" , True ), + ("mode" , "pyLoad + plugins;plugins only", "Check updates for" , "pyLoad + plugins"), + ("interval" , "int" , "Check interval in hours" , 8 ), + ("autorestart" , "bool" , "Automatically restart pyLoad when required" , True ), + ("reloadplugins", "bool" , "Monitor plugins for code changes in debug mode", True ), + ("nodebugupdate", "bool" , "Don't check for updates in debug mode" , False )] + + __description__ = """Check for updates""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + # event_list = ["pluginConfigChanged"] + + SERVER_URL = "http://updatemanager.pyload.org" + VERSION = re.compile(r'__version__.*=.*("|\')([\d.]+)') + MIN_INTERVAL = 3 * 60 * 60 #: 3h minimum check interval (value is in seconds) + + + def pluginConfigChanged(self, plugin, name, value): + if name == "interval": + interval = value * 60 * 60 + if self.MIN_INTERVAL <= interval != self.interval: + self.core.scheduler.removeJob(self.cb) + self.interval = interval + self.initPeriodical() + else: + self.logDebug("Invalid interval value, kept current") + + elif name == "reloadplugins": + if self.cb2: + self.core.scheduler.removeJob(self.cb2) + if value is True and self.core.debug: + self.periodical2() + + + def activate(self): + self.pluginConfigChanged(self.__name__, "interval", self.getConfig("interval")) + x = lambda: self.pluginConfigChanged(self.__name__, "reloadplugins", self.getConfig("reloadplugins")) + self.core.scheduler.addJob(10, x, threaded=False) + + + def deactivate(self): + self.pluginConfigChanged(self.__name__, "reloadplugins", False) + + + def setup(self): + self.cb2 = None + self.interval = 0 + self.updating = False + self.info = {'pyload': False, 'version': None, 'plugins': False} + self.mtimes = {} #: store modification time for each plugin + + + def periodical2(self): + if not self.updating: + self.autoreloadPlugins() + + self.cb2 = self.core.scheduler.addJob(4, self.periodical2, threaded=False) + + + @Expose + def autoreloadPlugins(self): + """ reload and reindex all modified plugins """ + modules = filter( + lambda m: m and (m.__name__.startswith("pyload.plugin.") or + m.__name__.startswith("userplugins.")) and + m.__name__.count(".") >= 2, sys.modules.itervalues() + ) + + reloads = [] + + for m in modules: + root, type, name = m.__name__.rsplit(".", 2) + id = (type, name) + if type in self.core.pluginManager.plugins: + f = m.__file__.replace(".pyc", ".py") + if not path.isfile(f): + continue + + mtime = stat(f).st_mtime + + if id not in self.mtimes: + self.mtimes[id] = mtime + elif self.mtimes[id] < mtime: + reloads.append(id) + self.mtimes[id] = mtime + + return True if self.core.pluginManager.reloadPlugins(reloads) else False + + + def periodical(self): + if self.info['pyload'] or self.getConfig("nodebugupdate") and self.core.debug: + return + + self.updateThread() + + + def server_request(self): + try: + return getURL(self.SERVER_URL, get={'v': self.core.api.getServerVersion()}).splitlines() + except Exception: + self.logWarning(_("Unable to contact server to get updates")) + + + @threaded + def updateThread(self): + self.updating = True + + status = self.update(onlyplugin=self.getConfig("mode") == "plugins only") + + if status is 2 and self.getConfig("autorestart"): + self.core.api.restart() + else: + self.updating = False + + + @Expose + def updatePlugins(self): + """ simple wrapper for calling plugin update quickly """ + return self.update(onlyplugin=True) + + + @Expose + def update(self, onlyplugin=False): + """ check for updates """ + data = self.server_request() + + if not data: + exitcode = 0 + + elif data[0] == "None": + self.logInfo(_("No new pyLoad version available")) + updates = data[1:] + exitcode = self._updatePlugins(updates) + + elif onlyplugin: + exitcode = 0 + + else: + newversion = data[0] + self.logInfo(_("*** New pyLoad Version %s available ***") % newversion) + self.logInfo(_("*** Get it here: https://github.com/pyload/pyload/releases ***")) + exitcode = 3 + self.info['pyload'] = True + self.info['version'] = newversion + + return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required; 3 = No plugins updated, new pyLoad version available + + + def _updatePlugins(self, updates): + """ check for plugin updates """ + + if self.info['plugins']: + return False #: plugins were already updated + + exitcode = 0 + updated = [] + + url = updates[0] + schema = updates[1].split('|') + + if "BLACKLIST" in updates: + blacklist = updates[updates.index('BLACKLIST') + 1:] + updates = updates[2:updates.index('BLACKLIST')] + else: + blacklist = None + updates = updates[2:] + + upgradable = [dict(zip(schema, x.split('|'))) for x in updates] + blacklisted = [(x.split('|')[0], x.split('|')[1].rsplit('.', 1)[0]) for x in blacklist] if blacklist else [] + + if blacklist: + # Protect UpdateManager from self-removing + try: + blacklisted.remove(("internal", "UpdateManager")) + except Exception: + pass + + for t, n in blacklisted: + for idx, plugin in enumerate(upgradable): + if n == plugin['name'] and t == plugin['type']: + upgradable.pop(idx) + break + + for t, n in self.removePlugins(sorted(blacklisted)): + self.logInfo(_("Removed blacklisted plugin [%(type)s] %(name)s") % { + 'type': t, + 'name': n, + }) + + for plugin in sorted(upgradable, key=itemgetter("type", "name")): + filename = plugin['name'] + type = plugin['type'] + version = plugin['version'] + + if filename.endswith(".pyc"): + name = filename[:filename.find("_")] + else: + name = filename.replace(".py", "") + + plugins = getattr(self.core.pluginManager, "%sPlugins" % type) + + oldver = float(plugins[name]['version']) if name in plugins else None + newver = float(version) + + if not oldver: + msg = "New plugin: [%(type)s] %(name)s (v%(newver).2f)" + elif newver > oldver: + msg = "New version of plugin: [%(type)s] %(name)s (v%(oldver).2f -> v%(newver).2f)" + else: + continue + + self.logInfo(_(msg) % {'type' : type, + 'name' : name, + 'oldver': oldver, + 'newver': newver}) + try: + content = getURL(url % plugin) + m = self.VERSION.search(content) + + if m and m.group(2) == version: + with open(safe_join("userplugins", prefix, filename), "wb") as f: + f.write(content) + + updated.append((prefix, name)) + else: + raise Exception, _("Version mismatch") + + except Exception, e: + self.logError(_("Error updating plugin: %s") % filename, str(e)) + + if updated: + reloaded = self.core.pluginManager.reloadPlugins(updated) + if reloaded: + self.logInfo(_("Plugins updated and reloaded")) + exitcode = 1 + else: + self.logInfo(_("*** Plugins have been updated, but need a pyLoad restart to be reloaded ***")) + self.info['plugins'] = True + exitcode = 2 + else: + self.logInfo(_("No plugin updates available")) + + return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required + + + @Expose + def removePlugins(self, type_plugins): + """ delete plugins from disk """ + + if not type_plugins: + return + + self.logDebug("Requested deletion of plugins: %s" % type_plugins) + + removed = [] + + for type, name in type_plugins: + err = False + file = name + ".py" + + for root in ("userplugins", path.join(pypath, "pyload", "plugins")): + + filename = safe_join(root, type, file) + try: + remove(filename) + except Exception, e: + self.logDebug("Error deleting: %s" % path.basename(filename), e) + err = True + + filename += "c" + if path.isfile(filename): + try: + if type == "addon": + self.manager.deactivateAddon(name) + remove(filename) + except Exception, e: + self.logDebug("Error deleting: %s" % path.basename(filename), e) + err = True + + if not err: + id = (type, name) + removed.append(id) + + return removed #: return a list of the plugins successfully removed diff --git a/pyload/plugin/internal/XFSAccount.py b/pyload/plugin/internal/XFSAccount.py new file mode 100644 index 000000000..2e6b7dc50 --- /dev/null +++ b/pyload/plugin/internal/XFSAccount.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- + +import re + +from time import gmtime, mktime, strptime +from urlparse import urljoin + +from pyload.plugin.Account import Account +from pyload.plugin.internal.SimpleHoster import parseHtmlForm, set_cookies + + +class XFSAccount(Account): + __name__ = "XFSAccount" + __type__ = "account" + __version__ = "0.36" + + __description__ = """XFileSharing account plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = None + HOSTER_URL = None + LOGIN_URL = None + + COOKIES = True + + PREMIUM_PATTERN = r'\(Premium only\)' + + VALID_UNTIL_PATTERN = r'Premium.[Aa]ccount expire:.*?(\d{1,2} [\w^_]+ \d{4})' + + TRAFFIC_LEFT_PATTERN = r'Traffic available today:.*?\s*(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' + TRAFFIC_LEFT_UNIT = "MB" #: used only if no group was found + + LEECH_TRAFFIC_PATTERN = r'Leech Traffic left:.*?(?P[\d.,]+|[Uu]nlimited)\s*(?:(?P[\w^_]+)\s*)?' + LEECH_TRAFFIC_UNIT = "MB" #: used only if no group was found + + LOGIN_FAIL_PATTERN = r'Incorrect Login or Password|account was banned|Error<' + + + def init(self): + if not self.HOSTER_DOMAIN: + self.logError(_("Missing HOSTER_DOMAIN")) + self.COOKIES = False + + else: + if not self.HOSTER_URL: + self.HOSTER_URL = "http://www.%s/" % self.HOSTER_DOMAIN + + if isinstance(self.COOKIES, list): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + set_cookies(req.cj, self.COOKIES) + + + def loadAccountInfo(self, user, req): + validuntil = None + trafficleft = None + leechtraffic = None + premium = None + + if not self.HOSTER_URL: #@TODO: Remove in 0.4.10 + return {'validuntil' : validuntil, + 'trafficleft' : trafficleft, + 'leechtraffic': leechtraffic, + 'premium' : premium} + + html = req.load(self.HOSTER_URL, get={'op': "my_account"}, decode=True) + + premium = True if re.search(self.PREMIUM_PATTERN, html) else False + + m = re.search(self.VALID_UNTIL_PATTERN, html) + if m: + expiredate = m.group(1).strip() + self.logDebug("Expire date: " + expiredate) + + try: + validuntil = mktime(strptime(expiredate, "%d %B %Y")) + + except Exception, e: + self.logError(e) + + else: + self.logDebug("Valid until: %s" % validuntil) + + if validuntil > mktime(gmtime()): + premium = True + trafficleft = -1 + else: + premium = False + validuntil = None #: registered account type (not premium) + else: + self.logDebug("VALID_UNTIL_PATTERN not found") + + m = re.search(self.TRAFFIC_LEFT_PATTERN, html) + if m: + try: + traffic = m.groupdict() + size = traffic['S'] + + if "nlimited" in size: + trafficleft = -1 + if validuntil is None: + validuntil = -1 + else: + if 'U' in traffic: + unit = traffic['U'] + elif isinstance(self.TRAFFIC_LEFT_UNIT, basestring): + unit = self.TRAFFIC_LEFT_UNIT + else: + unit = "" + + trafficleft = self.parseTraffic(size + unit) + + except Exception, e: + self.logError(e) + else: + self.logDebug("TRAFFIC_LEFT_PATTERN not found") + + leech = [m.groupdict() for m in re.finditer(self.LEECH_TRAFFIC_PATTERN, html)] + if leech: + leechtraffic = 0 + try: + for traffic in leech: + size = traffic['S'] + + if "nlimited" in size: + leechtraffic = -1 + if validuntil is None: + validuntil = -1 + break + else: + if 'U' in traffic: + unit = traffic['U'] + elif isinstance(self.LEECH_TRAFFIC_UNIT, basestring): + unit = self.LEECH_TRAFFIC_UNIT + else: + unit = "" + + leechtraffic += self.parseTraffic(size + unit) + + except Exception, e: + self.logError(e) + else: + self.logDebug("LEECH_TRAFFIC_PATTERN not found") + + return {'validuntil' : validuntil, + 'trafficleft' : trafficleft, + 'leechtraffic': leechtraffic, + 'premium' : premium} + + + def login(self, user, data, req): + if not self.HOSTER_URL: #@TODO: Remove in 0.4.10 + raise Exception(_("Missing HOSTER_DOMAIN")) + + if not self.LOGIN_URL: + self.LOGIN_URL = urljoin(self.HOSTER_URL, "login.html") + html = req.load(self.LOGIN_URL, decode=True) + + action, inputs = parseHtmlForm('name="FL"', html) + if not inputs: + inputs = {'op' : "login", + 'redirect': self.HOSTER_URL} + + inputs.update({'login' : user, + 'password': data['password']}) + + if not action: + action = self.HOSTER_URL + html = req.load(action, post=inputs, decode=True) + + if re.search(self.LOGIN_FAIL_PATTERN, html): + self.wrongPassword() diff --git a/pyload/plugin/internal/XFSCrypter.py b/pyload/plugin/internal/XFSCrypter.py new file mode 100644 index 000000000..1a03b69d0 --- /dev/null +++ b/pyload/plugin/internal/XFSCrypter.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +from pyload.plugin.internal.SimpleCrypter import SimpleCrypter + + +class XFSCrypter(SimpleCrypter): + __name__ = "XFSCrypter" + __type__ = "crypter" + __version__ = "0.06" + + __pattern__ = r'^unmatchable$' + + __description__ = """XFileSharing decrypter plugin""" + __license__ = "GPLv3" + __authors__ = [("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = None + + URL_REPLACEMENTS = [(r'&?per_page=\d+', ""), (r'[?/&]+$', ""), (r'(.+/[^?]+)$', r'\1?'), (r'$', r'&per_page=10000')] + + LINK_PATTERN = r'<(?:td|TD).*?>\s*.+?(?:)?\s*' + NAME_PATTERN = r'<[tT]itle>.*?\: (?P.+) folder' + + OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' + + + def prepare(self): + if not self.HOSTER_DOMAIN: + if self.account: + account = self.account + else: + account_name = (self.__name__ + ".py").replace("Folder.py", "").replace(".py", "") + account = self.pyfile.m.core.accountManager.getAccountPlugin(account_name) + + if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: + self.HOSTER_DOMAIN = account.HOSTER_DOMAIN + else: + self.fail(_("Missing HOSTER_DOMAIN")) + + if isinstance(self.COOKIES, list): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + + return super(XFSCrypter, self).prepare() diff --git a/pyload/plugin/internal/XFSHoster.py b/pyload/plugin/internal/XFSHoster.py new file mode 100644 index 000000000..b0a5aff0f --- /dev/null +++ b/pyload/plugin/internal/XFSHoster.py @@ -0,0 +1,326 @@ +# -*- coding: utf-8 -*- + +import re + +from random import random +from time import sleep +from urlparse import urljoin, urlparse + +from pyload.plugin.internal.captcha import ReCaptcha, SolveMedia +from pyload.plugin.internal.SimpleHoster import SimpleHoster, secondsToMidnight +from pyload.utils import html_unescape + + +class XFSHoster(SimpleHoster): + __name__ = "XFSHoster" + __type__ = "hoster" + __version__ = "0.44" + + __pattern__ = r'^unmatchable$' + + __description__ = """XFileSharing hoster plugin""" + __license__ = "GPLv3" + __authors__ = [("zoidberg", "zoidberg@mujmail.cz"), + ("stickell", "l.stickell@yahoo.it"), + ("Walter Purcaro", "vuolter@gmail.com")] + + + HOSTER_DOMAIN = None + + TEXT_ENCODING = False + DIRECT_LINK = None + MULTI_HOSTER = True #@NOTE: Should be default to False for safe, but I'm lazy... + + NAME_PATTERN = r'(Filename[ ]*:[ ]*()?|name="fname"[ ]+value="|<[\w^_]+ class="(file)?name">)\s*(?P.+?)(\s*<|")' + SIZE_PATTERN = r'(Size[ ]*:[ ]*()?|File:.*>|\s*\(|<[\w^_]+ class="size">)\s*(?P[\d.,]+)\s*(?P[\w^_]+)' + + OFFLINE_PATTERN = r'>\s*\w+ (Not Found|file (was|has been) removed)' + TEMP_OFFLINE_PATTERN = r'>\s*\w+ server (is in )?(maintenance|maintainance)' + + WAIT_PATTERN = r'.*?>(\d+)|id="countdown" value=".*?(\d+).*?"' + PREMIUM_ONLY_PATTERN = r'>This file is available for Premium Users only' + ERROR_PATTERN = r'(?:class=["\']err["\'].*?>|<[Cc]enter>|>Error|>\(ERROR:)(?:\s*<.+?>\s*)*(.+?)(?:["\']|<|\))' + + LINK_LEECH_PATTERN = r'

Download Link

\s*]*>([^<]+)' + LINK_PATTERN = None #: final download url pattern + + CAPTCHA_PATTERN = r'(https?://[^"\']+?/captchas?/[^"\']+)' + CAPTCHA_BLOCK_PATTERN = r'>Enter code.*?(.+?)
' + RECAPTCHA_PATTERN = None + SOLVEMEDIA_PATTERN = None + + FORM_PATTERN = None + FORM_INPUTS_MAP = None #: dict passed as input_names to parseHtmlForm + + + def setup(self): + self.chunkLimit = -1 if self.premium else 1 + self.resumeDownload = self.multiDL = self.premium + + + def prepare(self): + """ Initialize important variables """ + if not self.HOSTER_DOMAIN: + if self.account: + account = self.account + else: + account = self.pyfile.m.core.accountManager.getAccountPlugin(self.__name__) + + if account and hasattr(account, "HOSTER_DOMAIN") and account.HOSTER_DOMAIN: + self.HOSTER_DOMAIN = account.HOSTER_DOMAIN + else: + self.fail(_("Missing HOSTER_DOMAIN")) + + if isinstance(self.COOKIES, list): + self.COOKIES.insert((self.HOSTER_DOMAIN, "lang", "english")) + + if not self.LINK_PATTERN: + pattern = r'(https?://(?:www\.)?([^/]*?%s|\d+\.\d+\.\d+\.\d+)(\:\d+)?(/d/|(/files)?/\d+/\w+/).+?)["\'<]' + self.LINK_PATTERN = pattern % self.HOSTER_DOMAIN.replace('.', '\.') + + self.captcha = None + self.errmsg = None + + super(XFSHoster, self).prepare() + + if self.DIRECT_LINK is None: + self.directDL = self.premium + + + def handleFree(self, pyfile): + for i in xrange(1, 6): + self.logDebug("Getting download link: #%d" % i) + + self.checkErrors() + + m = re.search(self.LINK_PATTERN, self.html, re.S) + if m: + break + + data = self.getPostParameters() + + self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True, follow_location=False) + + m = re.search(r'Location\s*:\s*(.+)', self.req.http.header, re.I) + if m and not "op=" in m.group(1): + break + + m = re.search(self.LINK_PATTERN, self.html, re.S) + if m: + break + else: + self.logError(data['op'] if 'op' in data else _("UNKNOWN")) + return "" + + self.link = m.group(1).strip() #@TODO: Remove .strip() in 0.4.10 + + + def handlePremium(self, pyfile): + return self.handleFree(pyfile) + + + def handleMulti(self, pyfile): + if not self.account: + self.fail(_("Only registered or premium users can use url leech feature")) + + #only tested with easybytez.com + self.html = self.load("http://www.%s/" % self.HOSTER_DOMAIN) + + action, inputs = self.parseHtmlForm() + + upload_id = "%012d" % int(random() * 10 ** 12) + action += upload_id + "&js_on=1&utype=prem&upload_type=url" + + inputs['tos'] = '1' + inputs['url_mass'] = pyfile.url + inputs['up1oad_type'] = 'url' + + self.logDebug(action, inputs) + + self.req.setOption("timeout", 600) #: wait for file to upload to easybytez.com + + self.html = self.load(action, post=inputs) + + self.checkErrors() + + action, inputs = self.parseHtmlForm('F1') + if not inputs: + if self.errmsg: + self.retry(reason=self.errmsg) + else: + self.error(_("TEXTAREA F1 not found")) + + self.logDebug(inputs) + + stmsg = inputs['st'] + + if stmsg == 'OK': + self.html = self.load(action, post=inputs) + + elif 'Can not leech file' in stmsg: + self.retry(20, 3 * 60, _("Can not leech file")) + + elif 'today' in stmsg: + self.retry(wait_time=secondsToMidnight(gmt=2), reason=_("You've used all Leech traffic today")) + + else: + self.fail(stmsg) + + #get easybytez.com link for uploaded file + m = re.search(self.LINK_LEECH_PATTERN, self.html) + if m is None: + self.error(_("LINK_LEECH_PATTERN not found")) + + header = self.load(m.group(1), just_header=True, decode=True) + + if 'location' in header: #: Direct download link + self.link = header['location'] + + + def checkErrors(self): + m = re.search(self.ERROR_PATTERN, self.html) + if m is None: + self.errmsg = None + else: + self.errmsg = m.group(1).strip() + + self.logWarning(re.sub(r"<.*?>", " ", self.errmsg)) + + if 'wait' in self.errmsg: + wait_time = sum(int(v) * {"hr": 3600, "hour": 3600, "min": 60, "sec": 1}[u.lower()] for v, u in + re.findall(r'(\d+)\s*(hr|hour|min|sec)', self.errmsg, re.I)) + self.wait(wait_time, True) + + elif 'country' in self.errmsg: + self.fail(_("Downloads are disabled for your country")) + + elif 'captcha' in self.errmsg: + self.invalidCaptcha() + + elif 'premium' in self.errmsg and 'require' in self.errmsg: + self.fail(_("File can be downloaded by premium users only")) + + elif 'limit' in self.errmsg: + if 'days' in self.errmsg: + delay = secondsToMidnight(gmt=2) + retries = 3 + else: + delay = 1 * 60 * 60 + retries = 24 + + self.wantReconnect = True + self.retry(retries, delay, _("Download limit exceeded")) + + elif 'countdown' in self.errmsg or 'Expired' in self.errmsg: + self.retry(reason=_("Link expired")) + + elif 'maintenance' in self.errmsg or 'maintainance' in self.errmsg: + self.tempOffline() + + elif 'up to' in self.errmsg: + self.fail(_("File too large for free download")) + + else: + self.wantReconnect = True + self.retry(wait_time=60, reason=self.errmsg) + + if self.errmsg: + self.info['error'] = self.errmsg + else: + self.info.pop('error', None) + + + def getPostParameters(self): + if self.FORM_PATTERN or self.FORM_INPUTS_MAP: + action, inputs = self.parseHtmlForm(self.FORM_PATTERN or "", self.FORM_INPUTS_MAP or {}) + else: + action, inputs = self.parseHtmlForm(input_names={'op': re.compile(r'^download')}) + + if not inputs: + action, inputs = self.parseHtmlForm('F1') + if not inputs: + if self.errmsg: + self.retry(reason=self.errmsg) + else: + self.error(_("TEXTAREA F1 not found")) + + self.logDebug(inputs) + + if 'op' in inputs: + if "password" in inputs: + password = self.getPassword() + if password: + inputs['password'] = password + else: + self.fail(_("Missing password")) + + if not self.premium: + m = re.search(self.WAIT_PATTERN, self.html) + if m: + wait_time = int(m.group(1)) + self.setWait(wait_time, False) + + self.captcha = self.handleCaptcha(inputs) + + self.wait() + else: + inputs['referer'] = self.pyfile.url + + if self.premium: + inputs['method_premium'] = "Premium Download" + inputs.pop('method_free', None) + else: + inputs['method_free'] = "Free Download" + inputs.pop('method_premium', None) + + return inputs + + + def handleCaptcha(self, inputs): + m = re.search(self.CAPTCHA_PATTERN, self.html) + if m: + captcha_url = m.group(1) + inputs['code'] = self.decryptCaptcha(captcha_url) + return 1 + + m = re.search(self.CAPTCHA_BLOCK_PATTERN, self.html, re.S) + if m: + captcha_div = m.group(1) + numerals = re.findall(r'(\d)', html_unescape(captcha_div)) + + self.logDebug(captcha_div) + + inputs['code'] = "".join(a[1] for a in sorted(numerals, key=lambda num: int(num[0]))) + + self.logDebug("Captcha code: %s" % inputs['code'], numerals) + return 2 + + recaptcha = ReCaptcha(self) + try: + captcha_key = re.search(self.RECAPTCHA_PATTERN, self.html).group(1) + + except Exception: + captcha_key = recaptcha.detect_key() + + else: + self.logDebug("ReCaptcha key: %s" % captcha_key) + + if captcha_key: + inputs['recaptcha_response_field'], inputs['recaptcha_challenge_field'] = recaptcha.challenge(captcha_key) + return 3 + + solvemedia = SolveMedia(self) + try: + captcha_key = re.search(self.SOLVEMEDIA_PATTERN, self.html).group(1) + + except Exception: + captcha_key = solvemedia.detect_key() + + else: + self.logDebug("SolveMedia key: %s" % captcha_key) + + if captcha_key: + inputs['adcopy_response'], inputs['adcopy_challenge'] = solvemedia.challenge(captcha_key) + return 4 + + return 0 diff --git a/pyload/plugin/internal/__init__.py b/pyload/plugin/internal/__init__.py new file mode 100644 index 000000000..40a96afc6 --- /dev/null +++ b/pyload/plugin/internal/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- -- cgit v1.2.3