diff options
| -rw-r--r-- | module/Api.py | 24 | ||||
| -rw-r--r-- | module/CaptchaManager.py | 16 | ||||
| -rw-r--r-- | module/HookManager.py | 19 | ||||
| -rw-r--r-- | module/PluginThread.py | 59 | ||||
| -rw-r--r-- | module/PyFile.py | 2 | ||||
| -rw-r--r-- | module/common/packagetools.py | 132 | ||||
| -rw-r--r-- | module/network/HTTPChunk.py | 4 | ||||
| -rw-r--r-- | module/network/HTTPDownload.py | 9 | ||||
| -rw-r--r-- | module/network/HTTPRequest.py | 60 | ||||
| -rw-r--r-- | module/network/RequestFactory.py | 15 | ||||
| -rw-r--r-- | module/network/XDCCRequest.py | 2 | ||||
| -rw-r--r-- | module/utils.py | 6 | ||||
| -rw-r--r-- | module/web/cnl_app.py | 47 | ||||
| -rw-r--r-- | module/web/pyload_app.py | 2 |
14 files changed, 186 insertions, 211 deletions
diff --git a/module/Api.py b/module/Api.py index 551085b51..f0bf5e264 100644 --- a/module/Api.py +++ b/module/Api.py @@ -22,13 +22,11 @@ from os.path import join from time import time import re -from urlparse import urlparse - from PyFile import PyFile +from utils import freeSpace, compare_time from common.packagetools import parseNames from network.RequestFactory import getURL from remote import activated -from utils import compare_time, freeSpace, html_unescape, save_path if activated: try: @@ -51,7 +49,7 @@ def permission(bits): def __new__(cls, func, *args, **kwargs): permMap[func.__name__] = bits return func - + return _Dec @@ -162,7 +160,7 @@ class Api(Iface): @permission(PERMS.SETTINGS) def getConfig(self): """Retrieves complete config of core. - + :return: list of `ConfigSection` """ return self._convertConfigFormat(self.core.config.config) @@ -221,7 +219,7 @@ class Api(Iface): @permission(PERMS.LIST) def statusServer(self): """Some general information about the current status of pyLoad. - + :return: `ServerStatus` """ serverStatus = ServerStatus(self.core.threadManager.pause, len(self.core.threadManager.processingIds()), @@ -319,11 +317,11 @@ class Api(Iface): :return: package id of the new package """ if self.core.config['general']['folder_per_package']: - folder = urlparse(html_unescape(name)).path.split("/")[-1] + folder = name else: folder = "" - folder = save_path(folder) + folder = folder.replace("http://", "").replace(":", "").replace("/", "_").replace("\\", "_") pid = self.core.files.addPackage(name, folder, dest) @@ -486,7 +484,7 @@ class Api(Iface): :return: `PackageData` with .fid attribute """ data = self.core.files.getPackageData(int(pid)) - + if not data: raise PackageDoesNotExists(pid) @@ -513,7 +511,7 @@ class Api(Iface): @permission(PERMS.DELETE) def deleteFiles(self, fids): """Deletes several file entries from pyload. - + :param fids: list of file ids """ for id in fids: @@ -586,7 +584,7 @@ class Api(Iface): @permission(PERMS.ADD) def addFiles(self, pid, links): """Adds files to specific package. - + :param pid: package id :param links: list of urls """ @@ -708,7 +706,7 @@ class Api(Iface): """Gives a package a new position. :param pid: package id - :param position: + :param position: """ self.core.files.reorderPackage(pid, position) @@ -920,7 +918,7 @@ class Api(Iface): :param username: :param password: - :param remoteip: + :param remoteip: :return: dict with info, empty when login is incorrect """ if self.core.config["remote"]["nolocalauth"] and remoteip == "127.0.0.1": diff --git a/module/CaptchaManager.py b/module/CaptchaManager.py index b7b81a5b7..02cd10a11 100644 --- a/module/CaptchaManager.py +++ b/module/CaptchaManager.py @@ -58,11 +58,11 @@ class CaptchaManager(): self.lock.release() return None - def handleCaptcha(self, task, timeout=50): + def handleCaptcha(self, task): cli = self.core.isClientConnected() if cli: #client connected -> should solve the captcha - task.setWaiting(timeout) #wait 50 sec for response + task.setWaiting(50) #wait 50 sec for response for plugin in self.core.hookManager.activePlugins(): try: @@ -125,10 +125,10 @@ class CaptchaTask(): self.status = "waiting" def isWaiting(self): - if self.result or self.error or self.timedOut(): + if self.result or self.error or time() > self.waitUntil: return False - else: - return True + + return True def isTextual(self): """ returns if text is written on the captcha """ @@ -149,12 +149,10 @@ class CaptchaTask(): def invalid(self): """ indicates the captcha was not correct """ - for x in self.handler: - x.captchaInvalid(self) + [x.captchaInvalid(self) for x in self.handler] def correct(self): - for x in self.handler: - x.captchaCorrect(self) + [x.captchaCorrect(self) for x in self.handler] def __str__(self): return "<CaptchaTask '%s'>" % self.id diff --git a/module/HookManager.py b/module/HookManager.py index e4b9a33a0..16f692d76 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -141,8 +141,8 @@ class HookManager: if self.core.debug: traceback.print_exc() - self.log.info(_("Activated addons: %s") % ", ".join(sorted(active))) - self.log.info(_("Deactivated addons: %s") % ", ".join(sorted(deactive))) + self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active))) + self.log.info(_("Deactivate plugins: %s") % ", ".join(sorted(deactive))) self.plugins = plugins @@ -219,7 +219,10 @@ class HookManager: def downloadFinished(self, pyfile): for plugin in self.plugins: if plugin.isActivated(): - plugin.downloadFinished(pyfile) + if "downloadFinished" in plugin.__threaded__: + self.startThread(plugin.downloadFinished, pyfile) + else: + plugin.downloadFinished(pyfile) self.dispatchEvent("downloadFinished", pyfile) @@ -228,7 +231,10 @@ class HookManager: def downloadFailed(self, pyfile): for plugin in self.plugins: if plugin.isActivated(): - plugin.downloadFailed(pyfile) + if "downloadFailed" in plugin.__threaded__: + self.startThread(plugin.downloadFinished, pyfile) + else: + plugin.downloadFailed(pyfile) self.dispatchEvent("downloadFailed", pyfile) @@ -236,7 +242,10 @@ class HookManager: def packageFinished(self, package): for plugin in self.plugins: if plugin.isActivated(): - plugin.packageFinished(package) + if "packageFinished" in plugin.__threaded__: + self.startThread(plugin.packageFinished, package) + else: + plugin.packageFinished(package) self.dispatchEvent("packageFinished", package) diff --git a/module/PluginThread.py b/module/PluginThread.py index 051236c3e..56c36c778 100644 --- a/module/PluginThread.py +++ b/module/PluginThread.py @@ -155,7 +155,7 @@ class DownloadThread(PluginThread): """Constructor""" PluginThread.__init__(self, manager) - self.queue = Queue() #: job queue + self.queue = Queue() # job queue self.active = False self.start() @@ -176,8 +176,7 @@ class DownloadThread(PluginThread): return True try: - if not pyfile.hasPlugin(): - continue + if not pyfile.hasPlugin(): continue #this pyfile was deleted while queueing pyfile.plugin.checkForSameFiles(starting=True) @@ -185,7 +184,6 @@ class DownloadThread(PluginThread): # start download self.m.core.hookManager.downloadPreparing(pyfile) - pyfile.error = "" pyfile.plugin.preprocessing(self) self.m.log.info(_("Download finished: %s") % pyfile.name) @@ -207,9 +205,6 @@ class DownloadThread(PluginThread): pyfile.setStatus("aborted") - if self.m.core.debug: - print_exc() - self.clean(pyfile) continue @@ -242,9 +237,6 @@ class DownloadThread(PluginThread): self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) pyfile.error = msg - if self.m.core.debug: - print_exc() - self.m.core.hookManager.downloadFailed(pyfile) self.clean(pyfile) continue @@ -325,7 +317,7 @@ class DownloadThread(PluginThread): pyfile.checkIfProcessed() exc_clear() - + #pyfile.plugin.req.clean() self.active = False @@ -367,46 +359,41 @@ class DecrypterThread(PluginThread): retry = False try: - self.m.log.info(_("Decrypting starts: %s") % pyfile.name) - pyfile.error = "" - pyfile.plugin.preprocessing(self) + self.m.log.info(_("Decrypting starts: %s") % self.active.name) + self.active.plugin.preprocessing(self) except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) return except Fail, e: msg = e.args[0] if msg == "offline": - pyfile.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % pyfile.name) + self.active.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % self.active.name) else: - pyfile.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) - pyfile.error = msg + self.active.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": msg}) + self.active.error = msg - if self.m.core.debug: - print_exc() return except Abort: self.m.log.info(_("Download aborted: %s") % pyfile.name) pyfile.setStatus("aborted") - if self.m.core.debug: - print_exc() return except Retry: - self.m.log.info(_("Retrying %s") % pyfile.name) + self.m.log.info(_("Retrying %s") % self.active.name) retry = True return self.run() except Exception, e: - pyfile.setStatus("failed") - self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) - pyfile.error = str(e) + self.active.setStatus("failed") + self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % {"name": self.active.name, "msg": str(e)}) + self.active.error = str(e) if self.m.core.debug: print_exc() @@ -414,14 +401,21 @@ class DecrypterThread(PluginThread): return + finally: if not retry: - pyfile.release() + self.active.release() self.active = False self.m.core.files.save() self.m.localThreads.remove(self) exc_clear() + + #self.m.core.hookManager.downloadFinished(pyfile) + + + #self.m.localThreads.remove(self) + #self.active.finishIfDone() if not retry: pyfile.delete() @@ -467,7 +461,7 @@ class HookThread(PluginThread): #dirty method to filter out exceptions if "unexpected keyword argument 'thread'" not in e.args[0]: raise - + del self.kwargs["thread"] self.f(*self.args, **self.kwargs) finally: @@ -630,10 +624,9 @@ class InfoThread(PluginThread): self.m.log.debug("Run Info Fetching for %s" % pluginname) for result in plugin.getInfo(process): #result = [ .. (name, size, status, url) .. ] - if not type(result) == list: - result = [result] + if not type(result) == list: result = [result] - for res in result: #: why don't assign res dict directly? + for res in result: self.m.infoCache[res[3]] = res cb(pluginname, result) diff --git a/module/PyFile.py b/module/PyFile.py index 9e20ceab4..3dede9360 100644 --- a/module/PyFile.py +++ b/module/PyFile.py @@ -251,7 +251,7 @@ class PyFile(object): def getBytesLeft(self): """ gets bytes left """ try: - return self.getSize() - self.plugin.req.arrived + return self.plugin.req.size - self.plugin.req.arrived except: return 0 diff --git a/module/common/packagetools.py b/module/common/packagetools.py index d930157e1..5bfbcba95 100644 --- a/module/common/packagetools.py +++ b/module/common/packagetools.py @@ -1,57 +1,17 @@ -# -*- coding: utf-8 -*- +#!/usr/bin/env python -import re +# JDownloader/src/jd/controlling/LinkGrabberPackager.java +import re from urlparse import urlparse - -endings = ("jdeatme", "3gp", "7zip", "7z", "abr", "ac3", "aiff", "aifc", "aif", "ai", - "au", "avi", "apk", "bin", "bmp", "bat", "bz2", "cbr", "cbz", "ccf", "chm", - "cr2", "cso", "cue", "cvd", "dta", "deb", "divx", "djvu", "dlc", "dmg", "doc", - "docx", "dot", "eps", "epub", "exe", "ff", "flv", "flac", "f4v", "gsd", "gif", - "gpg", "gz", "iwd", "idx", "iso", "ipa", "ipsw", "java", "jar", "jpe?g", "load", - "m2ts", "m4v", "m4a", "md5", "mkv", "mp2", "mp3", "mp4", "mobi", "mov", "movie", - "mpeg", "mpe", "mpg", "mpq", "msi", "msu", "msp", "mv", "mws", "nfo", "npk", "oga", - "ogg", "ogv", "otrkey", "par2", "pkg", "png", "pdf", "pptx?", "ppsx?", "ppz", "pot", - "psd", "qt", "rmvb", "rm", "rar", "ram", "ra", "rev", "rnd", "rpm", "run", "rsdf", - "reg", "rtf", "shnf", "sh(?!tml)", "ssa", "smi", "sub", "srt", "snd", "sfv", "sfx", - "swf", "swc", "tar\.(gz|bz2|xz)", "tar", "tgz", "tiff?", "ts", "txt", "viv", "vivo", - "vob", "vtt", "webm", "wav", "wmv", "wma", "xla", "xls", "xpi", "zeno", "zip", - "[r-z]\d{2}", "_[_a-z]{2}", "\d{3,4}(?=\?|$|\"|\r|\n)") - -rarPats = [re.compile(r'(.*)(\.|_|-)pa?r?t?\.?\d+.(rar|exe)$', re.I), - re.compile(r'(.*)(\.|_|-)part\.?[0]*[1].(rar|exe)$', re.I), - re.compile(r'(.*)\.rar$', re.I), - re.compile(r'(.*)\.r\d+$', re.I), - re.compile(r'(.*)(\.|_|-)\d+$', re.I)] - -zipPats = [re.compile(r'(.*)\.zip$', re.I), - re.compile(r'(.*)\.z\d+$', re.I), - re.compile(r'(?is).*\.7z\.[\d]+$', re.I), - re.compile(r'(.*)\.a.$', re.I)] - -ffsjPats = [re.compile(r'(.*)\._((_[a-z])|([a-z]{2}))(\.|$)'), - re.compile(r'(.*)(\.|_|-)[\d]+(\.(' + '|'.join(endings) + ')$)', re.I)] - -iszPats = [re.compile(r'(.*)\.isz$', re.I), - re.compile(r'(.*)\.i\d{2}$', re.I)] - -pat0 = re.compile(r'www\d*\.', re.I) - -pat1 = re.compile(r'(\.?CD\d+)', re.I) -pat2 = re.compile(r'(\.?part\d+)', re.I) - -pat3 = re.compile(r'(.+)[\.\-_]+$') -pat4 = re.compile(r'(.+)\.\d+\.xtm$') - - def matchFirst(string, *args): - """ matches against list of regexp and returns first match """ + """ matches against list of regexp and returns first match""" for patternlist in args: for pattern in patternlist: - m = pattern.search(string) - if m is not None: - name = m.group(1) + r = pattern.search(string) + if r is not None: + name = r.group(1) return name return string @@ -61,10 +21,35 @@ def parseNames(files): """ Generates packages names from name, data lists :param files: list of (name, data) - :return: packagenames mapped to data lists (eg. urls) + :return: packagenames mapt to data lists (eg. urls) """ packs = {} + endings = "\\.(3gp|7zip|7z|abr|ac3|aiff|aifc|aif|ai|au|avi|bin|bz2|cbr|cbz|ccf|cue|cvd|chm|dta|deb|divx|djvu|dlc|dmg|doc|docx|dot|eps|exe|ff|flv|f4v|gsd|gif|gz|iwd|iso|ipsw|java|jar|jpg|jpeg|jdeatme|load|mws|mw|m4v|m4a|mkv|mp2|mp3|mp4|mov|movie|mpeg|mpe|mpg|msi|msu|msp|nfo|npk|oga|ogg|ogv|otrkey|pkg|png|pdf|pptx|ppt|pps|ppz|pot|psd|qt|rmvb|rm|rar|ram|ra|rev|rnd|r\\d+|rpm|run|rsdf|rtf|sh(!?tml)|srt|snd|sfv|swf|tar|tif|tiff|ts|txt|viv|vivo|vob|wav|wmv|xla|xls|xpi|zeno|zip|z\\d+|_[_a-z]{2}|\\d+$)" + + rarPats = [re.compile("(.*)(\\.|_|-)pa?r?t?\\.?[0-9]+.(rar|exe)$", re.I), + re.compile("(.*)(\\.|_|-)part\\.?[0]*[1].(rar|exe)$", re.I), + re.compile("(.*)\\.rar$", re.I), + re.compile("(.*)\\.r\\d+$", re.I), + re.compile("(.*)(\\.|_|-)\\d+$", re.I)] + + zipPats = [re.compile("(.*)\\.zip$", re.I), + re.compile("(.*)\\.z\\d+$", re.I), + re.compile("(?is).*\\.7z\\.[\\d]+$", re.I), + re.compile("(.*)\\.a.$", re.I)] + + ffsjPats = [re.compile("(.*)\\._((_[a-z])|([a-z]{2}))(\\.|$)"), + re.compile("(.*)(\\.|_|-)[\\d]+(" + endings + "$)", re.I)] + + iszPats = [re.compile("(.*)\\.isz$", re.I), + re.compile("(.*)\\.i\\d{2}$", re.I)] + + pat1 = re.compile("(\\.?CD\\d+)", re.I) + pat2 = re.compile("(\\.?part\\d+)", re.I) + + pat3 = re.compile("(.+)[\\.\\-_]+$") + pat4 = re.compile("(.+)\\.\\d+\\.xtm$") + for file, url in files: patternMatch = False @@ -79,7 +64,7 @@ def parseNames(files): if len(split) > 1: name = split.pop(1) - #check if an already existing package may be ok for this file + #check if a already existing package may be ok for this file # found = False # for pack in packs: # if pack in file: @@ -87,8 +72,7 @@ def parseNames(files): # found = True # break # - # if found: - # continue + # if found: continue # unrar pattern, 7zip/zip and hjmerge pattern, isz pattern, FFSJ pattern before = name @@ -97,19 +81,19 @@ def parseNames(files): patternMatch = True # xtremsplit pattern - m = pat4.search(name) - if m is not None: - name = m.group(1) + r = pat4.search(name) + if r is not None: + name = r.group(1) # remove part and cd pattern - m = pat1.search(name) - if m is not None: - name = name.replace(m.group(0), "") + r = pat1.search(name) + if r is not None: + name = name.replace(r.group(0), "") patternMatch = True - m = pat2.search(name) - if m is not None: - name = name.replace(m.group(0), "") + r = pat2.search(name) + if r is not None: + name = name.replace(r.group(0), "") patternMatch = True # additional checks if extension pattern matched @@ -124,9 +108,9 @@ def parseNames(files): name = name[:-length] # remove endings like . _ - - m = pat3.search(name) - if m is not None: - name = m.group(1) + r = pat3.search(name) + if r is not None: + name = r.group(1) # replace . and _ with space name = name.replace(".", " ") @@ -139,12 +123,11 @@ def parseNames(files): # fallback: package by hoster if not name: name = urlparse(file).hostname - if name: - name = pat0.sub("", name) + if name: name = name.replace("www.", "") # fallback : default name if not name: - name = _("Unnamed package") + name = "unknown" # build mapping if name in packs: @@ -153,3 +136,20 @@ def parseNames(files): packs[name] = [url] return packs + + +if __name__ == "__main__": + from os.path import join + from pprint import pprint + + f = open(join("..", "..", "testlinks2.txt"), "rb") + urls = [(x.strip(), x.strip()) for x in f.readlines() if x.strip()] + f.close() + + print "Having %d urls." % len(urls) + + packs = parseNames(urls) + + pprint(packs) + + print "Got %d urls." % sum([len(x) for x in packs.itervalues()]) diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index d42744cf4..b637aef32 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -208,7 +208,7 @@ class HTTPChunk(HTTPRequest): # as first chunk, we will parse the headers if not self.range and self.header.endswith("\r\n\r\n"): self.parseHeader() - elif not self.range and buf.startswith("150") and "data connection" in buf.lower(): #: ftp file size parsing + elif not self.range and buf.startswith("150") and "data connection" in buf: #ftp file size parsing size = search(r"(\d+) bytes", buf) if size: self.p.size = int(size.group(1)) @@ -290,4 +290,4 @@ class HTTPChunk(HTTPRequest): """ closes everything, unusable after this """ if self.fp: self.fp.close() self.c.close() - if hasattr(self, "p"): del self.p + if hasattr(self, "p"): del self.p
\ No newline at end of file diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index 3f32295b4..fe8075539 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -63,7 +63,7 @@ class HTTPDownload(): except IOError: self.info = ChunkInfo(filename) - self.chunkSupport = True + self.chunkSupport = None self.m = pycurl.CurlMulti() #needed for speed calculation @@ -130,7 +130,7 @@ class HTTPDownload(): except pycurl.error, e: #code 33 - no resume code = e.args[0] - if resume is True and code == 33: + if code == 33: # try again without resume self.log.debug("Errno 33 -> Restart without resume") @@ -151,7 +151,6 @@ class HTTPDownload(): if not resume: self.info.clear() self.info.addChunk("%s.chunk0" % self.filename, (0, 0)) #create an initial entry - self.info.save() self.chunks = [] @@ -165,8 +164,8 @@ class HTTPDownload(): chunksDone = set() # list of curl handles that are finished chunksCreated = False done = False - if self.info.getCount() is 0: # This is a resume, if we were chunked originally assume still can - self.chunkSupport = False + if self.info.getCount() > 1: # This is a resume, if we were chunked originally assume still can + self.chunkSupport = True while 1: #need to create chunks diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 67ede2854..4747d937f 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, see <http://www.gnu.org/licenses/>. - + @author: RaNaN """ @@ -25,11 +25,11 @@ from httplib import responses from logging import getLogger from cStringIO import StringIO -from module.plugins.Plugin import Abort, Fail +from module.plugins.Plugin import Abort def myquote(url): return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") - + def myurlencode(data): data = dict(data) return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ @@ -79,7 +79,7 @@ class HTTPRequest(): if hasattr(pycurl, "AUTOREFERER"): self.c.setopt(pycurl.AUTOREFERER, 1) self.c.setopt(pycurl.SSL_VERIFYPEER, 0) - self.c.setopt(pycurl.LOW_SPEED_TIME, 60) + self.c.setopt(pycurl.LOW_SPEED_TIME, 30) self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5) #self.c.setopt(pycurl.VERBOSE, 1) @@ -181,7 +181,7 @@ class HTTPRequest(): self.getCookies() - def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False, follow_location=True, save_cookies=True): + def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False): """ load and returns a given page """ self.setRequestContext(url, get, post, referer, cookies, multipart) @@ -190,27 +190,24 @@ class HTTPRequest(): self.c.setopt(pycurl.HTTPHEADER, self.headers) - if not follow_location: - self.c.setopt(pycurl.FOLLOWLOCATION, 0) - if just_header: + self.c.setopt(pycurl.FOLLOWLOCATION, 0) self.c.setopt(pycurl.NOBODY, 1) + self.c.perform() + rep = self.header - self.c.perform() - rep = self.header if just_header else self.getResponse() - - if not follow_location: self.c.setopt(pycurl.FOLLOWLOCATION, 1) - - if just_header: self.c.setopt(pycurl.NOBODY, 0) + else: + self.c.perform() + rep = self.getResponse() + self.c.setopt(pycurl.POSTFIELDS, "") self.lastEffectiveURL = self.c.getinfo(pycurl.EFFECTIVE_URL) self.code = self.verifyHeader() - if save_cookies: - self.addCookies() + self.addCookies() if decode: rep = self.decodeResponse(rep) @@ -231,13 +228,11 @@ class HTTPRequest(): def getResponse(self): """ retrieve response from string io """ - if self.rep is None: - return "" - else: - value = self.rep.getvalue() - self.rep.close() - self.rep = StringIO() - return value + if self.rep is None: return "" + value = self.rep.getvalue() + self.rep.close() + self.rep = StringIO() + return value def decodeResponse(self, rep): """ decode with correct encoding, relies on header """ @@ -260,7 +255,7 @@ class HTTPRequest(): #self.log.debug("Decoded %s" % encoding ) if lookup(encoding).name == 'utf-8' and rep.startswith(BOM_UTF8): encoding = 'utf-8-sig' - + decoder = getincrementaldecoder(encoding)("replace") rep = decoder.decode(rep, True) @@ -268,7 +263,6 @@ class HTTPRequest(): except LookupError: self.log.debug("No Decoder foung for %s" % encoding) - except Exception: self.log.debug("Error when decoding string from %s." % encoding) @@ -278,15 +272,13 @@ class HTTPRequest(): """ writes response """ if self.rep.tell() > 1000000 or self.abort: rep = self.getResponse() + if self.abort: raise Abort() + f = open("response.dump", "wb") + f.write(rep) + f.close() + raise Exception("Loaded Url exceeded limit") - if self.abort: - raise Abort() - - with open("response.dump", "wb") as f: - f.write(rep) - raise Fail("Loaded url exceeded size limit") - else: - self.rep.write(buf) + self.rep.write(buf) def writeHeader(self, buf): """ writes header """ @@ -311,4 +303,4 @@ if __name__ == "__main__": url = "http://pyload.org" c = HTTPRequest() print c.load(url) - + diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 750f37dc9..5b1528281 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -62,23 +62,12 @@ class RequestFactory(): def getURL(self, *args, **kwargs): """ see HTTPRequest for argument list """ - cj = None - - if 'cookies' in kwargs: - if isinstance(kwargs['cookies'], CookieJar): - cj = kwargs['cookies'] - elif isinstance(kwargs['cookies'], list): - cj = CookieJar(None) - for cookie in kwargs['cookies']: - if isinstance(cookie, tuple) and len(cookie) == 3: - cj.setCookie(*cookie) - - h = HTTPRequest(cj, self.getOptions()) + h = HTTPRequest(None, self.getOptions()) try: rep = h.load(*args, **kwargs) finally: h.close() - + return rep def getCookieJar(self, pluginName, account=None): diff --git a/module/network/XDCCRequest.py b/module/network/XDCCRequest.py index e395e01f0..f03798c17 100644 --- a/module/network/XDCCRequest.py +++ b/module/network/XDCCRequest.py @@ -127,7 +127,7 @@ class XDCCRequest(): return filename - def _keepAlive(self, sock, *readbuffer): + def _keepAlive(self, sock, readbuffer): fdset = select([sock], [], [], 0) if sock not in fdset[0]: return diff --git a/module/utils.py b/module/utils.py index 4406eafde..8748b7693 100644 --- a/module/utils.py +++ b/module/utils.py @@ -36,9 +36,9 @@ def remove_chars(string, repl): def save_path(name): #remove some chars if os.name == 'nt': - return remove_chars(name, '/?%*|"<>') + return remove_chars(name, '/\\?%*:|"<>') else: - return remove_chars(name, '\\"') + return remove_chars(name, '/\\"') def save_join(*args): @@ -54,7 +54,7 @@ if sys.getfilesystemencoding().startswith('ANSI'): try: string = string.encode('utf-8') finally: - return save_path(string) + return string fs_decode = decode #decode utf8 diff --git a/module/web/cnl_app.py b/module/web/cnl_app.py index 718499dfb..d8f7c1180 100644 --- a/module/web/cnl_app.py +++ b/module/web/cnl_app.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*- from os.path import join import re @@ -8,7 +9,6 @@ from binascii import unhexlify from bottle import route, request, HTTPError from webinterface import PYLOAD, DL_ROOT, JS - try: from Crypto.Cipher import AES except: @@ -17,8 +17,8 @@ except: def local_check(function): def _view(*args, **kwargs): - if request.environ.get("REMOTE_ADDR", "0") in ("127.0.0.1", "localhost") \ - or request.environ.get("HTTP_HOST", "0") in ("127.0.0.1:9666", "localhost:9666"): + if request.environ.get('REMOTE_ADDR', "0") in ('127.0.0.1', 'localhost') \ + or request.environ.get('HTTP_HOST','0') == '127.0.0.1:9666': return function(*args, **kwargs) else: return HTTPError(403, "Forbidden") @@ -26,15 +26,14 @@ def local_check(function): return _view -@route('/flash') -@route('/flash/<id>') -@route('/flash', method='POST') +@route("/flash") +@route("/flash/:id") +@route("/flash", method="POST") @local_check def flash(id="0"): return "JDownloader\r\n" - -@route('/flash/add', method='POST') +@route("/flash/add", method="POST") @local_check def add(request): package = request.POST.get('referer', None) @@ -47,10 +46,10 @@ def add(request): return "" - -@route('/flash/addcrypted', method='POST') +@route("/flash/addcrypted", method="POST") @local_check def addcrypted(): + package = request.forms.get('referer', 'ClickAndLoad Package') dlc = request.forms['crypted'].replace(" ", "+") @@ -66,10 +65,10 @@ def addcrypted(): else: return "success\r\n" - -@route('/flash/addcrypted2', method='POST') +@route("/flash/addcrypted2", method="POST") @local_check def addcrypted2(): + package = request.forms.get("source", None) crypted = request.forms["crypted"] jk = request.forms["jk"] @@ -83,7 +82,7 @@ def addcrypted2(): try: jk = re.findall(r"return ('|\")(.+)('|\")", jk)[0][1] except: - ## Test for some known js functions to decode + ## Test for some known js functions to decode if jk.find("dec") > -1 and jk.find("org") > -1: org = re.findall(r"var org = ('|\")([^\"']+)", jk)[0][1] jk = list(org) @@ -101,7 +100,7 @@ def addcrypted2(): IV = Key obj = AES.new(Key, AES.MODE_CBC, IV) - result = obj.decrypt(crypted).replace("\x00", "").replace("\r", "").split("\n") + result = obj.decrypt(crypted).replace("\x00", "").replace("\r","").split("\n") result = filter(lambda x: x != "", result) @@ -115,14 +114,13 @@ def addcrypted2(): else: return "success\r\n" - -@route('/flashgot_pyload') -@route('/flashgot_pyload', method='POST') -@route('/flashgot') -@route('/flashgot', method='POST') +@route("/flashgot_pyload") +@route("/flashgot_pyload", method="POST") +@route("/flashgot") +@route("/flashgot", method="POST") @local_check def flashgot(): - if request.environ['HTTP_REFERER'] not in ("http://localhost:9666/flashgot", "http://127.0.0.1:9666/flashgot"): + if request.environ['HTTP_REFERER'] != "http://localhost:9666/flashgot" and request.environ['HTTP_REFERER'] != "http://127.0.0.1:9666/flashgot": return HTTPError() autostart = int(request.forms.get('autostart', 0)) @@ -137,8 +135,7 @@ def flashgot(): return "" - -@route('/crossdomain.xml') +@route("/crossdomain.xml") @local_check def crossdomain(): rep = "<?xml version=\"1.0\"?>\n" @@ -149,17 +146,17 @@ def crossdomain(): return rep -@route('/flash/checkSupportForUrl') +@route("/flash/checkSupportForUrl") @local_check def checksupport(): + url = request.GET.get("url") res = PYLOAD.checkURLs([url]) supported = (not res[0][1] is None) return str(supported).lower() - -@route('/jdcheck.js') +@route("/jdcheck.js") @local_check def jdcheck(): rep = "jdownloader=true;\n" diff --git a/module/web/pyload_app.py b/module/web/pyload_app.py index f3515f05f..df4a4b3d4 100644 --- a/module/web/pyload_app.py +++ b/module/web/pyload_app.py @@ -261,7 +261,7 @@ def config(): elif not data.trafficleft: data.trafficleft = _("not available") else: - data.trafficleft = formatSize(data.trafficleft) + data.trafficleft = formatSize(data.trafficleft * 1024) if data.validuntil == -1: data.validuntil = _("unlimited") |