summaryrefslogtreecommitdiffstats
path: root/pyload/plugins/internal
diff options
context:
space:
mode:
Diffstat (limited to 'pyload/plugins/internal')
-rw-r--r--pyload/plugins/internal/UpdateManager.py281
-rw-r--r--pyload/plugins/internal/XFileSharingPro.py352
2 files changed, 633 insertions, 0 deletions
diff --git a/pyload/plugins/internal/UpdateManager.py b/pyload/plugins/internal/UpdateManager.py
new file mode 100644
index 000000000..eb9411236
--- /dev/null
+++ b/pyload/plugins/internal/UpdateManager.py
@@ -0,0 +1,281 @@
+# -*- coding: utf-8 -*-
+
+import re
+import sys
+
+from operator import itemgetter
+from os import path, remove, stat
+
+from pyload.network.RequestFactory import getURL
+from pyload.plugins.Hook import Expose, Hook, threaded
+from pyload.utils import safe_join
+
+
+class UpdateManager(Hook):
+ __name__ = "UpdateManager"
+ __type__ = "hook"
+ __version__ = "0.35"
+
+ __config__ = [("activated", "bool", "Activated", True),
+ ("mode", "pyLoad + plugins;plugins only", "Check updates for", "pyLoad + plugins"),
+ ("interval", "int", "Check interval in hours", 8),
+ ("reloadplugins", "bool", "Monitor plugins for code changes (debug mode only)", True),
+ ("nodebugupdate", "bool", "Don't check for updates in debug mode", True)]
+
+ __description__ = """ Check for updates """
+ __author_name__ = "Walter Purcaro"
+ __author_mail__ = "vuolter@gmail.com"
+
+
+ event_list = ["pluginConfigChanged"]
+
+ SERVER_URL = "http://updatemanager.pyload.org"
+ MIN_INTERVAL = 3 * 60 * 60 #: 3h minimum check interval (value is in seconds)
+
+
+ def pluginConfigChanged(self, plugin, name, value):
+ if name == "interval":
+ interval = value * 60 * 60
+ if self.MIN_INTERVAL <= interval != self.interval:
+ self.core.scheduler.removeJob(self.cb)
+ self.interval = interval
+ self.initPeriodical()
+ else:
+ self.logDebug("Invalid interval value, kept current")
+ elif name == "reloadplugins":
+ if self.cb2:
+ self.core.scheduler.removeJob(self.cb2)
+ if value is True and self.core.debug:
+ self.periodical2()
+
+ def coreReady(self):
+ self.pluginConfigChanged(self.__name__, "interval", self.getConfig("interval"))
+ x = lambda: self.pluginConfigChanged(self.__name__, "reloadplugins", self.getConfig("reloadplugins"))
+ self.core.scheduler.addJob(10, x, threaded=False)
+
+ def unload(self):
+ self.pluginConfigChanged(self.__name__, "reloadplugins", False)
+
+ def setup(self):
+ self.cb2 = None
+ self.interval = self.MIN_INTERVAL
+ self.updating = False
+ self.info = {'pyload': False, 'version': None, 'plugins': False}
+ self.mtimes = {} #: store modification time for each plugin
+
+ def periodical2(self):
+ if not self.updating:
+ self.autoreloadPlugins()
+ self.cb2 = self.core.scheduler.addJob(4, self.periodical2, threaded=False)
+
+ @Expose
+ def autoreloadPlugins(self):
+ """ reload and reindex all modified plugins """
+ modules = filter(
+ lambda m: m and (m.__name__.startswith("pyload.plugins.") or
+ m.__name__.startswith("userplugins.")) and
+ m.__name__.count(".") >= 2, sys.modules.itervalues()
+ )
+
+ reloads = []
+
+ for m in modules:
+ root, type, name = m.__name__.rsplit(".", 2)
+ id = (type, name)
+ if type in self.core.pluginManager.plugins:
+ f = m.__file__.replace(".pyc", ".py")
+ if not path.isfile(f):
+ continue
+
+ mtime = stat(f).st_mtime
+
+ if id not in self.mtimes:
+ self.mtimes[id] = mtime
+ elif self.mtimes[id] < mtime:
+ reloads.append(id)
+ self.mtimes[id] = mtime
+
+ return True if self.core.pluginManager.reloadPlugins(reloads) else False
+
+ def periodical(self):
+ if not self.info['pyload'] and not (self.getConfig("nodebugupdate") and self.core.debug):
+ self.updateThread()
+
+ def server_request(self):
+ try:
+ return getURL(self.SERVER_URL, get={'v': self.core.api.getServerVersion()}).splitlines()
+ except:
+ self.logWarning(_("Unable to contact server to get updates"))
+
+ @threaded
+ def updateThread(self):
+ self.updating = True
+ status = self.update(onlyplugin=self.getConfig("mode") == "plugins only")
+ if status == 2:
+ self.core.api.restart()
+ else:
+ self.updating = False
+
+ @Expose
+ def updatePlugins(self):
+ """ simple wrapper for calling plugin update quickly """
+ return self.update(onlyplugin=True)
+
+ @Expose
+ def update(self, onlyplugin=False):
+ """ check for updates """
+ data = self.server_request()
+ if not data:
+ exitcode = 0
+ elif data[0] == "None":
+ self.logInfo(_("No new pyLoad version available"))
+ updates = data[1:]
+ exitcode = self._updatePlugins(updates)
+ elif onlyplugin:
+ exitcode = 0
+ else:
+ newversion = data[0]
+ self.logInfo(_("*** New pyLoad Version %s available ***") % newversion)
+ self.logInfo(_("*** Get it here: https://github.com/pyload/pyload/releases ***"))
+ exitcode = 3
+ self.info['pyload'] = True
+ self.info['version'] = newversion
+ return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required; 3 = No plugins updated, new pyLoad version available
+
+ def _updatePlugins(self, updates):
+ """ check for plugin updates """
+
+ if self.info['plugins']:
+ return False #: plugins were already updated
+
+ updated = []
+
+ vre = re.compile(r'__version__.*=.*("|\')([0-9.]+)')
+ url = updates[0]
+ schema = updates[1].split('|')
+ if "BLACKLIST" in updates:
+ blacklist = updates[updates.index('BLACKLIST') + 1:]
+ updates = updates[2:updates.index('BLACKLIST')]
+ else:
+ blacklist = None
+ updates = updates[2:]
+
+ upgradable = sorted(map(lambda x: dict(zip(schema, x.split('|'))), updates), key=itemgetter("type", "name"))
+ for plugin in upgradable:
+ filename = plugin['name']
+ prefix = plugin['type']
+ version = plugin['version']
+
+ if filename.endswith(".pyc"):
+ name = filename[:filename.find("_")]
+ else:
+ name = filename.replace(".py", "")
+
+ #@TODO: obsolete after 0.4.10
+ if prefix.endswith("s"):
+ type = prefix[:-1]
+ else:
+ type = prefix
+
+ plugins = getattr(self.core.pluginManager, "%sPlugins" % type)
+
+ oldver = float(plugins[name]['v']) if name in plugins else None
+ newver = float(version)
+
+ if not oldver:
+ msg = "New [%(type)s] %(name)s (v%(newver)s)"
+ elif newver > oldver:
+ msg = "New version of [%(type)s] %(name)s (v%(oldver)s -> v%(newver)s)"
+ else:
+ continue
+
+ self.logInfo(_(msg) % {
+ 'type': type,
+ 'name': name,
+ 'oldver': oldver,
+ 'newver': newver,
+ })
+
+ try:
+ content = getURL(url % plugin)
+ m = vre.search(content)
+ if m and m.group(2) == version:
+ f = open(safe_join("userplugins", prefix, filename), "wb")
+ f.write(content)
+ f.close()
+ updated.append((prefix, name))
+ else:
+ raise Exception, _("Version mismatch")
+ except Exception, e:
+ self.logError(_("Error updating plugin %s") % filename, e)
+
+ if blacklist:
+ blacklisted = sorted(map(lambda x: (x.split('|')[0], x.split('|')[1].rsplit('.', 1)[0]), blacklist))
+
+ # Always protect UpdateManager from self-removing
+ try:
+ blacklisted.remove(("internal", "UpdateManager"))
+ except:
+ pass
+
+ removed = self.removePlugins(blacklisted)
+ for t, n in removed:
+ self.logInfo(_("Removed blacklisted plugin [%(type)s] %(name)s") % {
+ 'type': t,
+ 'name': n,
+ })
+
+ if updated:
+ reloaded = self.core.pluginManager.reloadPlugins(updated)
+ if reloaded:
+ self.logInfo(_("Plugins updated and reloaded"))
+ exitcode = 1
+ else:
+ self.logInfo(_("*** Plugins have been updated, but need a pyLoad restart to be reloaded ***"))
+ self.info['plugins'] = True
+ exitcode = 2
+ else:
+ self.logInfo(_("No plugin updates available"))
+ exitcode = 0
+
+ return exitcode #: 0 = No plugins updated; 1 = Plugins updated; 2 = Plugins updated, but restart required
+
+ @Expose
+ def removePlugins(self, type_plugins):
+ """ delete plugins from disk """
+
+ if not type_plugins:
+ return
+
+ self.logDebug("Requested deletion of plugins", type_plugins)
+
+ removed = []
+
+ for type, name in type_plugins:
+ err = False
+ file = name + ".py"
+
+ for root in ("userplugins", path.join(pypath, "pyload", "plugins")):
+
+ filename = safe_join(root, type, file)
+ try:
+ remove(filename)
+ except Exception, e:
+ self.logDebug("Error deleting", path.basename(filename), e)
+ err = True
+
+ filename += "c"
+ if path.isfile(filename):
+ try:
+ if type == "hook":
+ self.manager.deactivateHook(name)
+ remove(filename)
+ except Exception, e:
+ self.logDebug("Error deleting", path.basename(filename), e)
+ err = True
+
+ if not err:
+ id = (type, name)
+ removed.append(id)
+
+ return removed #: return a list of the plugins successfully removed
diff --git a/pyload/plugins/internal/XFileSharingPro.py b/pyload/plugins/internal/XFileSharingPro.py
new file mode 100644
index 000000000..212ef23ef
--- /dev/null
+++ b/pyload/plugins/internal/XFileSharingPro.py
@@ -0,0 +1,352 @@
+# -*- coding: utf-8 -*-
+
+import re
+
+from pycurl import FOLLOWLOCATION, LOW_SPEED_TIME
+from random import random
+from urllib import unquote
+from urlparse import urlparse
+
+from pyload.network.RequestFactory import getURL
+from pyload.plugins.internal.CaptchaService import ReCaptcha, SolveMedia
+from pyload.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo, PluginParseError, replace_patterns
+from pyload.utils import html_unescape
+
+
+class XFileSharingPro(SimpleHoster):
+ """
+ Common base for XFileSharingPro hosters like EasybytezCom, CramitIn, FiledinoCom...
+ Some hosters may work straight away when added to __pattern__
+ However, most of them will NOT work because they are either down or running a customized version
+ """
+ __name__ = "XFileSharingPro"
+ __type__ = "hoster"
+ __version__ = "0.36"
+
+ __pattern__ = r'^unmatchable$'
+
+ __description__ = """XFileSharingPro base hoster plugin"""
+ __author_name__ = ("zoidberg", "stickell", "Walter Purcaro")
+ __author_mail__ = ("zoidberg@mujmail.cz", "l.stickell@yahoo.it", "vuolter@gmail.com")
+
+
+ HOSTER_NAME = None
+
+ FILE_URL_REPLACEMENTS = [(r'/embed-(\w{12}).*', r'/\1')] #: support embedded files
+
+ COOKIES = [(HOSTER_NAME, "lang", "english")]
+
+ FILE_INFO_PATTERN = r'<tr><td align=right><b>Filename:</b></td><td nowrap>(?P<N>[^<]+)</td></tr>\s*.*?<small>\((?P<S>[^<]+)\)</small>'
+ FILE_NAME_PATTERN = r'<input type="hidden" name="fname" value="(?P<N>[^"]+)"'
+ FILE_SIZE_PATTERN = r'You have requested .*\((?P<S>[\d\.\,]+) ?(?P<U>\w+)?\)</font>'
+
+ OFFLINE_PATTERN = r'>\w+ (Not Found|file (was|has been) removed)'
+
+ WAIT_PATTERN = r'<span id="countdown_str">.*?>(\d+)</span>'
+
+ OVR_LINK_PATTERN = r'<h2>Download Link</h2>\s*<textarea[^>]*>([^<]+)'
+ LINK_PATTERN = None #: final download url pattern
+
+ CAPTCHA_URL_PATTERN = r'(http://[^"\']+?/captchas?/[^"\']+)'
+ RECAPTCHA_URL_PATTERN = r'http://[^"\']+?recaptcha[^"\']+?\?k=([^"\']+)"'
+ CAPTCHA_DIV_PATTERN = r'>Enter code.*?<div.*?>(.*?)</div>'
+ SOLVEMEDIA_PATTERN = r'http:\/\/api\.solvemedia\.com\/papi\/challenge\.script\?k=(.*?)"'
+
+ ERROR_PATTERN = r'class=["\']err["\'][^>]*>(.*?)</'
+
+
+ def setup(self):
+ self.chunkLimit = 1
+
+ if self.__name__ == "XFileSharingPro":
+ self.multiDL = True
+ self.__pattern__ = self.core.pluginManager.hosterPlugins[self.__name__]['pattern']
+ self.HOSTER_NAME = re.match(self.__pattern__, self.pyfile.url).group(1).lower()
+ self.COOKIES = [(self.HOSTER_NAME, "lang", "english")]
+ else:
+ self.resumeDownload = self.multiDL = self.premium
+
+
+ def prepare(self):
+ """ Initialize important variables """
+ if not self.HOSTER_NAME:
+ self.fail("Missing HOSTER_NAME")
+
+ if not self.LINK_PATTERN:
+ pattr = r'(http://([^/]*?%s|\d+\.\d+\.\d+\.\d+)(:\d+)?(/d/|(?:/files)?/\d+/\w+/)[^"\'<]+)'
+ self.LINK_PATTERN = pattr % self.HOSTER_NAME
+
+ if isinstance(self.COOKIES, list):
+ set_cookies(self.req.cj, self.COOKIES)
+
+ self.captcha = None
+ self.errmsg = None
+ self.passwords = self.getPassword().splitlines()
+
+
+ def process(self, pyfile):
+ self.prepare()
+
+ pyfile.url = replace_patterns(pyfile.url, self.FILE_URL_REPLACEMENTS)
+
+ if not re.match(self.__pattern__, pyfile.url):
+ if self.premium:
+ self.handleOverriden()
+ else:
+ self.fail("Only premium users can download from other hosters with %s" % self.HOSTER_NAME)
+ else:
+ try:
+ # Due to a 0.4.9 core bug self.load would use cookies even if
+ # cookies=False. Workaround using getURL to avoid cookies.
+ # Can be reverted in 0.4.10 as the cookies bug has been fixed.
+ self.html = getURL(pyfile.url, decode=True, cookies=self.COOKIES)
+ self.file_info = self.getFileInfo()
+ except PluginParseError:
+ self.file_info = None
+
+ self.location = self.getDirectDownloadLink()
+
+ if not self.file_info:
+ pyfile.name = html_unescape(unquote(urlparse(
+ self.location if self.location else pyfile.url).path.split("/")[-1]))
+
+ if self.location:
+ self.startDownload(self.location)
+ elif self.premium:
+ self.handlePremium()
+ else:
+ self.handleFree()
+
+
+ def getDirectDownloadLink(self):
+ """ Get download link for premium users with direct download enabled """
+ self.req.http.lastURL = self.pyfile.url
+
+ self.req.http.c.setopt(FOLLOWLOCATION, 0)
+ self.html = self.load(self.pyfile.url, decode=True)
+ self.header = self.req.http.header
+ self.req.http.c.setopt(FOLLOWLOCATION, 1)
+
+ location = None
+ m = re.search(r"Location\s*:\s*(.*)", self.header, re.I)
+ if m and re.match(self.LINK_PATTERN, m.group(1)):
+ location = m.group(1).strip()
+
+ return location
+
+
+ def handleFree(self):
+ url = self.getDownloadLink()
+ self.logDebug("Download URL: %s" % url)
+ self.startDownload(url)
+
+
+ def getDownloadLink(self):
+ for i in xrange(5):
+ self.logDebug("Getting download link: #%d" % i)
+ data = self.getPostParameters()
+
+ self.req.http.c.setopt(FOLLOWLOCATION, 0)
+ self.html = self.load(self.pyfile.url, post=data, ref=True, decode=True)
+ self.header = self.req.http.header
+ self.req.http.c.setopt(FOLLOWLOCATION, 1)
+
+ m = re.search(r"Location\s*:\s*(.*)", self.header, re.I)
+ if m:
+ break
+
+ m = re.search(self.LINK_PATTERN, self.html, re.S)
+ if m:
+ break
+
+ else:
+ if self.errmsg and 'captcha' in self.errmsg:
+ self.fail("No valid captcha code entered")
+ else:
+ self.fail("Download link not found")
+
+ return m.group(1)
+
+
+ def handlePremium(self):
+ self.html = self.load(self.pyfile.url, post=self.getPostParameters())
+ m = re.search(self.LINK_PATTERN, self.html)
+ if m is None:
+ self.parseError('DIRECT LINK')
+ self.startDownload(m.group(1))
+
+
+ def handleOverriden(self):
+ #only tested with easybytez.com
+ self.html = self.load("http://www.%s/" % self.HOSTER_NAME)
+ action, inputs = self.parseHtmlForm('')
+ upload_id = "%012d" % int(random() * 10 ** 12)
+ action += upload_id + "&js_on=1&utype=prem&upload_type=url"
+ inputs['tos'] = '1'
+ inputs['url_mass'] = self.pyfile.url
+ inputs['up1oad_type'] = 'url'
+
+ self.logDebug(self.HOSTER_NAME, action, inputs)
+ #wait for file to upload to easybytez.com
+ self.req.http.c.setopt(LOW_SPEED_TIME, 600)
+ self.html = self.load(action, post=inputs)
+
+ action, inputs = self.parseHtmlForm('F1')
+ if not inputs:
+ self.parseError('TEXTAREA')
+ self.logDebug(self.HOSTER_NAME, inputs)
+ if inputs['st'] == 'OK':
+ self.html = self.load(action, post=inputs)
+ elif inputs['st'] == 'Can not leech file':
+ self.retry(max_tries=20, wait_time=3 * 60, reason=inputs['st'])
+ else:
+ self.fail(inputs['st'])
+
+ #get easybytez.com link for uploaded file
+ m = re.search(self.OVR_LINK_PATTERN, self.html)
+ if m is None:
+ self.parseError('DIRECT LINK (OVR)')
+ self.pyfile.url = m.group(1)
+ header = self.load(self.pyfile.url, just_header=True)
+ if 'location' in header: # Direct link
+ self.startDownload(self.pyfile.url)
+ else:
+ self.retry()
+
+
+ def startDownload(self, link):
+ link = link.strip()
+ if self.captcha:
+ self.correctCaptcha()
+ self.logDebug("DIRECT LINK: %s" % link)
+ self.download(link, disposition=True)
+
+
+ def checkErrors(self):
+ m = re.search(self.ERROR_PATTERN, self.html)
+ if m:
+ self.errmsg = m.group(1)
+ self.logWarning(re.sub(r"<.*?>", " ", self.errmsg))
+
+ if 'wait' in self.errmsg:
+ wait_time = sum([int(v) * {"hour": 3600, "minute": 60, "second": 1}[u] for v, u in
+ re.findall(r'(\d+)\s*(hour|minute|second)', self.errmsg)])
+ self.wait(wait_time, True)
+ elif 'captcha' in self.errmsg:
+ self.invalidCaptcha()
+ elif 'premium' in self.errmsg and 'require' in self.errmsg:
+ self.fail("File can be downloaded by premium users only")
+ elif 'limit' in self.errmsg:
+ self.wait(1 * 60 * 60, True)
+ self.retry(25)
+ elif 'countdown' in self.errmsg or 'Expired' in self.errmsg:
+ self.retry()
+ elif 'maintenance' in self.errmsg:
+ self.tempOffline()
+ elif 'download files up to' in self.errmsg:
+ self.fail("File too large for free download")
+ else:
+ self.fail(self.errmsg)
+
+ else:
+ self.errmsg = None
+
+ return self.errmsg
+
+
+ def getPostParameters(self):
+ for _ in xrange(3):
+ if not self.errmsg:
+ self.checkErrors()
+
+ if hasattr(self, "FORM_PATTERN"):
+ action, inputs = self.parseHtmlForm(self.FORM_PATTERN)
+ else:
+ action, inputs = self.parseHtmlForm(input_names={"op": re.compile("^download")})
+
+ if not inputs:
+ action, inputs = self.parseHtmlForm('F1')
+ if not inputs:
+ if self.errmsg:
+ self.retry()
+ else:
+ self.parseError("Form not found")
+
+ self.logDebug(self.HOSTER_NAME, inputs)
+
+ if 'op' in inputs and inputs['op'] in ("download2", "download3"):
+ if "password" in inputs:
+ if self.passwords:
+ inputs['password'] = self.passwords.pop(0)
+ else:
+ self.fail("No or invalid passport")
+
+ if not self.premium:
+ m = re.search(self.WAIT_PATTERN, self.html)
+ if m:
+ wait_time = int(m.group(1)) + 1
+ self.setWait(wait_time, False)
+ else:
+ wait_time = 0
+
+ self.captcha = self.handleCaptcha(inputs)
+
+ if wait_time:
+ self.wait()
+
+ self.errmsg = None
+ return inputs
+
+ else:
+ inputs['referer'] = self.pyfile.url
+
+ if self.premium:
+ inputs['method_premium'] = "Premium Download"
+ if 'method_free' in inputs:
+ del inputs['method_free']
+ else:
+ inputs['method_free'] = "Free Download"
+ if 'method_premium' in inputs:
+ del inputs['method_premium']
+
+ self.html = self.load(self.pyfile.url, post=inputs, ref=True)
+ self.errmsg = None
+
+ else:
+ self.parseError('FORM: %s' % (inputs['op'] if 'op' in inputs else 'UNKNOWN'))
+
+
+ def handleCaptcha(self, inputs):
+ m = re.search(self.RECAPTCHA_URL_PATTERN, self.html)
+ if m:
+ recaptcha_key = unquote(m.group(1))
+ self.logDebug("RECAPTCHA KEY: %s" % recaptcha_key)
+ recaptcha = ReCaptcha(self)
+ inputs['recaptcha_challenge_field'], inputs['recaptcha_response_field'] = recaptcha.challenge(recaptcha_key)
+ return 1
+ else:
+ m = re.search(self.CAPTCHA_URL_PATTERN, self.html)
+ if m:
+ captcha_url = m.group(1)
+ inputs['code'] = self.decryptCaptcha(captcha_url)
+ return 2
+ else:
+ m = re.search(self.CAPTCHA_DIV_PATTERN, self.html, re.DOTALL)
+ if m:
+ captcha_div = m.group(1)
+ self.logDebug(captcha_div)
+ numerals = re.findall(r'<span.*?padding-left\s*:\s*(\d+).*?>(\d)</span>', html_unescape(captcha_div))
+ inputs['code'] = "".join([a[1] for a in sorted(numerals, key=lambda num: int(num[0]))])
+ self.logDebug("CAPTCHA", inputs['code'], numerals)
+ return 3
+ else:
+ m = re.search(self.SOLVEMEDIA_PATTERN, self.html)
+ if m:
+ captcha_key = m.group(1)
+ captcha = SolveMedia(self)
+ inputs['adcopy_challenge'], inputs['adcopy_response'] = captcha.challenge(captcha_key)
+ return 4
+ return 0
+
+
+getInfo = create_getInfo(XFileSharingPro)
generated by cgit v1.2.3 (git 2.25.1) at 2025-03-13 04:07:53 +0100