diff options
Diffstat (limited to 'module/network')
| -rw-r--r-- | module/network/Browser.py | 18 | ||||
| -rw-r--r-- | module/network/Bucket.py | 45 | ||||
| -rw-r--r-- | module/network/CookieJar.py | 66 | ||||
| -rw-r--r-- | module/network/HTTPChunk.py | 53 | ||||
| -rw-r--r-- | module/network/HTTPDownload.py | 66 | ||||
| -rw-r--r-- | module/network/HTTPRequest.py | 104 | ||||
| -rw-r--r-- | module/network/JsEngine.py | 256 | ||||
| -rw-r--r-- | module/network/RequestFactory.py | 45 | ||||
| -rw-r--r-- | module/network/XDCCRequest.py | 84 | ||||
| -rw-r--r-- | module/network/__init__.py | 2 |
10 files changed, 455 insertions, 284 deletions
diff --git a/module/network/Browser.py b/module/network/Browser.py index d68a23687..89341ff25 100644 --- a/module/network/Browser.py +++ b/module/network/Browser.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- from logging import getLogger -from HTTPRequest import HTTPRequest -from HTTPDownload import HTTPDownload +from pyload.network.HTTPRequest import HTTPRequest +from pyload.network.HTTPDownload import HTTPDownload class Browser(object): @@ -131,16 +130,3 @@ class Browser(object): del self.dl if hasattr(self, "cj"): del self.cj - -if __name__ == "__main__": - browser = Browser()#proxies={"socks5": "localhost:5000"}) - ip = "http://www.whatismyip.com/automation/n09230945.asp" - #browser.getPage("http://google.com/search?q=bar") - #browser.getPage("https://encrypted.google.com/") - #print browser.getPage(ip) - #print browser.getRedirectLocation("http://google.com/") - #browser.getPage("https://encrypted.google.com/") - #browser.getPage("http://google.com/search?q=bar") - - browser.httpDownload("http://speedtest.netcologne.de/test_10mb.bin", "test_10mb.bin") - diff --git a/module/network/Bucket.py b/module/network/Bucket.py index 69da277ae..408a1e240 100644 --- a/module/network/Bucket.py +++ b/module/network/Bucket.py @@ -1,45 +1,37 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" +# @author: RaNaN from time import time from threading import Lock -class Bucket: +MIN_RATE = 10240 #: 10kb minimum rate + + +class Bucket(object): + def __init__(self): - self.rate = 0 - self.tokens = 0 + self.rate = 0 # bytes per second, maximum targeted throughput + self.tokens = 0 self.timestamp = time() - self.lock = Lock() + self.lock = Lock() + def __nonzero__(self): - return False if self.rate < 10240 else True + return False if self.rate < MIN_RATE else True + def setRate(self, rate): self.lock.acquire() self.rate = int(rate) self.lock.release() + def consumed(self, amount): - """ return time the process have to sleep, after consumed specified amount """ - if self.rate < 10240: return 0 #min. 10kb, may become unresponsive otherwise - self.lock.acquire() + """ return the time the process has to sleep, after it consumed a specified amount """ + if self.rate < MIN_RATE: + return 0 #: May become unresponsive otherwise + self.lock.acquire() self.calc_tokens() self.tokens -= amount @@ -47,15 +39,14 @@ class Bucket: time = -self.tokens/float(self.rate) else: time = 0 - self.lock.release() return time + def calc_tokens(self): if self.tokens < self.rate: now = time() delta = self.rate * (now - self.timestamp) self.tokens = min(self.rate, self.tokens + delta) self.timestamp = now - diff --git a/module/network/CookieJar.py b/module/network/CookieJar.py index c05812334..a2b302776 100644 --- a/module/network/CookieJar.py +++ b/module/network/CookieJar.py @@ -1,50 +1,42 @@ # -*- coding: utf-8 -*- +# @author: RaNaN -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. +import Cookie - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. +from datetime import datetime, timedelta +from time import time - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - @author: mkaay, RaNaN -""" +# monkey patch for 32 bit systems +def _getdate(future=0, weekdayname=Cookie._weekdayname, monthname=Cookie._monthname): + dt = datetime.now() + timedelta(seconds=int(future)) + return "%s, %02d %3s %4d %02d:%02d:%02d GMT" % \ + (weekdayname[dt.weekday()], dt.day, monthname[dt.month], dt.year, dt.hour, dt.minute, dt.second) + +Cookie._getdate = _getdate -from time import time -class CookieJar(): - def __init__(self, pluginname, account=None): - self.cookies = {} - self.plugin = pluginname - self.account = account +class CookieJar(Cookie.SimpleCookie): - def addCookies(self, clist): - for c in clist: - name = c.split("\t")[5] - self.cookies[name] = c + def getCookie(self, name): + return self[name].value - def getCookies(self): - return self.cookies.values() + def setCookie(self, domain, name, value, path="/", exp=None, secure="FALSE"): + self[name] = value + self[name]["domain"] = domain + self[name]["path"] = path - def parseCookie(self, name): - if name in self.cookies: - return self.cookies[name].split("\t")[6] + # Value of expires should be integer if possible + # otherwise the cookie won't be used + if not exp: + expires = time() + 3600 * 24 * 180 else: - return None - - def getCookie(self, name): - return self.parseCookie(name) + try: + expires = int(exp) + except ValueError: + expires = exp - def setCookie(self, domain, name, value, path="/", exp=time()+3600*24*180): - s = ".%s TRUE %s FALSE %s %s %s" % (domain, path, exp, name, value) - self.cookies[name] = s + self[name]["expires"] = expires - def clear(self): - self.cookies = {} + if secure == "TRUE": + self[name]["secure"] = secure diff --git a/module/network/HTTPChunk.py b/module/network/HTTPChunk.py index b637aef32..41752b940 100644 --- a/module/network/HTTPChunk.py +++ b/module/network/HTTPChunk.py @@ -1,36 +1,22 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" +# @author: RaNaN + from os import remove, stat, fsync from os.path import exists from time import sleep from re import search -from module.utils import fs_encode +from pyload.utils import fs_encode import codecs import pycurl +import urllib -from HTTPRequest import HTTPRequest +from pyload.network.HTTPRequest import HTTPRequest class WrongFormat(Exception): pass -class ChunkInfo(): +class ChunkInfo(object): def __init__(self, name): self.name = unicode(name) self.size = 0 @@ -208,7 +194,7 @@ class HTTPChunk(HTTPRequest): # as first chunk, we will parse the headers if not self.range and self.header.endswith("\r\n\r\n"): self.parseHeader() - elif not self.range and buf.startswith("150") and "data connection" in buf: #ftp file size parsing + elif not self.range and buf.startswith("150") and "data connection" in buf.lower(): #: ftp file size parsing size = search(r"(\d+) bytes", buf) if size: self.p.size = int(size.group(1)) @@ -253,13 +239,23 @@ class HTTPChunk(HTTPRequest): """parse data from recieved header""" for orgline in self.decodeResponse(self.header).splitlines(): line = orgline.strip().lower() + if line.startswith("accept-ranges") and "bytes" in line: self.p.chunkSupport = True - if line.startswith("content-disposition") and "filename=" in line: - name = orgline.partition("filename=")[2] - name = name.replace('"', "").replace("'", "").replace(";", "").strip() + if line.startswith("content-disposition") and ("filename=" in line or "filename*=" in line): + if "filename*=" in line: + # extended version according to RFC 6266 and RFC 5987. + encoding = line.partition("*=")[2].partition("''")[0] + name = orgline.partition("''")[2] + name = urllib.unquote(str(name)).decode(charEnc(encoding)) + else: + # basic version, US-ASCII only + name = orgline.partition("filename=")[2] + + name = name.replace('"', "").replace("'", "").replace(";", "").replace("/", "_").strip() self.p.nameDisposition = name + self.log.debug("Content-Disposition: %s" % name) if not self.resume and line.startswith("content-length"): @@ -269,7 +265,7 @@ class HTTPChunk(HTTPRequest): def stop(self): """The download will not proceed after next call of writeBody""" - self.range = [0,0] + self.range = [0, 0] self.size = 0 def resetRange(self): @@ -290,4 +286,9 @@ class HTTPChunk(HTTPRequest): """ closes everything, unusable after this """ if self.fp: self.fp.close() self.c.close() - if hasattr(self, "p"): del self.p
\ No newline at end of file + if hasattr(self, "p"): del self.p + + +def charEnc(enc): + return {'utf-8' : "utf_8", + 'iso-8859-1': "latin_1"}.get(enc, "unknown") diff --git a/module/network/HTTPDownload.py b/module/network/HTTPDownload.py index fe8075539..3b2bf26ca 100644 --- a/module/network/HTTPDownload.py +++ b/module/network/HTTPDownload.py @@ -1,21 +1,5 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" +# @author: RaNaN from os import remove, fsync from os.path import dirname @@ -25,17 +9,18 @@ from logging import getLogger import pycurl -from HTTPChunk import ChunkInfo, HTTPChunk -from HTTPRequest import BadHeader +from pyload.network.HTTPChunk import ChunkInfo, HTTPChunk +from pyload.network.HTTPRequest import BadHeader -from module.plugins.Plugin import Abort -from module.utils import save_join, fs_encode +from pyload.plugin.Plugin import Abort +from pyload.utils import safe_join, fs_encode -class HTTPDownload(): + +class HTTPDownload(object): """ loads a url http + ftp """ def __init__(self, url, filename, get={}, post={}, referer=None, cj=None, bucket=None, - options={}, progressNotify=None, disposition=False): + options={}, progress=None, disposition=False): self.url = url self.filename = filename #complete file destination, not only name self.get = get @@ -63,7 +48,7 @@ class HTTPDownload(): except IOError: self.info = ChunkInfo(filename) - self.chunkSupport = None + self.chunkSupport = True self.m = pycurl.CurlMulti() #needed for speed calculation @@ -71,7 +56,7 @@ class HTTPDownload(): self.speeds = [] self.lastSpeeds = [0, 0] - self.progressNotify = progressNotify + self.progress = progress @property def speed(self): @@ -114,7 +99,7 @@ class HTTPDownload(): fo.close() if self.nameDisposition and self.disposition: - self.filename = save_join(dirname(self.filename), self.nameDisposition) + self.filename = safe_join(dirname(self.filename), self.nameDisposition) move(init, fs_encode(self.filename)) self.info.remove() #remove info file @@ -130,7 +115,7 @@ class HTTPDownload(): except pycurl.error, e: #code 33 - no resume code = e.args[0] - if code == 33: + if resume is True and code == 33: # try again without resume self.log.debug("Errno 33 -> Restart without resume") @@ -151,6 +136,7 @@ class HTTPDownload(): if not resume: self.info.clear() self.info.addChunk("%s.chunk0" % self.filename, (0, 0)) #create an initial entry + self.info.save() self.chunks = [] @@ -164,8 +150,8 @@ class HTTPDownload(): chunksDone = set() # list of curl handles that are finished chunksCreated = False done = False - if self.info.getCount() > 1: # This is a resume, if we were chunked originally assume still can - self.chunkSupport = True + if self.info.getCount() is 0: # This is a resume, if we were chunked originally assume still can + self.chunkSupport = False while 1: #need to create chunks @@ -284,9 +270,9 @@ class HTTPDownload(): self.updateProgress() if self.abort: - raise Abort() + raise Abort - #sleep(0.003) #supress busy waiting - limits dl speed to (1 / x) * buffersize + #sleep(0.003) #supress busy waiting - limits dl speed to (1 / x) * buffersize self.m.select(1) for chunk in self.chunks: @@ -295,8 +281,8 @@ class HTTPDownload(): self._copyChunks() def updateProgress(self): - if self.progressNotify: - self.progressNotify(self.percent) + if self.progress: + self.progress(self.percent) def findChunk(self, handle): """ linear search to find a chunk (should be ok since chunk size is usually low) """ @@ -324,17 +310,3 @@ class HTTPDownload(): del self.cj if hasattr(self, "info"): del self.info - -if __name__ == "__main__": - url = "http://speedtest.netcologne.de/test_100mb.bin" - - from Bucket import Bucket - - bucket = Bucket() - bucket.setRate(200 * 1024) - bucket = None - - print "starting" - - dwnld = HTTPDownload(url, "test_100mb.bin", bucket=bucket) - dwnld.download(chunks=3, resume=True) diff --git a/module/network/HTTPRequest.py b/module/network/HTTPRequest.py index 4747d937f..eac03a365 100644 --- a/module/network/HTTPRequest.py +++ b/module/network/HTTPRequest.py @@ -1,21 +1,7 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" +# @author: RaNaN + +from __future__ import with_statement import pycurl @@ -25,15 +11,17 @@ from httplib import responses from logging import getLogger from cStringIO import StringIO -from module.plugins.Plugin import Abort +from pyload.plugin.Plugin import Abort, Fail + +from pyload.utils import encode + def myquote(url): - return quote(url.encode('utf_8') if isinstance(url, unicode) else url, safe="%/:=&?~#+!$,;'@()*[]") - + return quote(encode(url), safe="%/:=&?~#+!$,;'@()*[]") + def myurlencode(data): data = dict(data) - return urlencode(dict((x.encode('utf_8') if isinstance(x, unicode) else x, \ - y.encode('utf_8') if isinstance(y, unicode) else y ) for x, y in data.iteritems())) + return urlencode(dict((encode(x), encode(y)) for x, y in data.iteritems())) bad_headers = range(400, 404) + range(405, 418) + range(500, 506) @@ -44,7 +32,7 @@ class BadHeader(Exception): self.content = content -class HTTPRequest(): +class HTTPRequest(object): def __init__(self, cookies=None, options=None): self.c = pycurl.Curl() self.rep = StringIO() @@ -79,8 +67,9 @@ class HTTPRequest(): if hasattr(pycurl, "AUTOREFERER"): self.c.setopt(pycurl.AUTOREFERER, 1) self.c.setopt(pycurl.SSL_VERIFYPEER, 0) - self.c.setopt(pycurl.LOW_SPEED_TIME, 30) + self.c.setopt(pycurl.LOW_SPEED_TIME, 60) self.c.setopt(pycurl.LOW_SPEED_LIMIT, 5) + self.c.setopt(pycurl.USE_SSL, pycurl.CURLUSESSL_TRY) #self.c.setopt(pycurl.VERBOSE, 1) @@ -89,8 +78,8 @@ class HTTPRequest(): if pycurl.version_info()[7]: self.c.setopt(pycurl.ENCODING, "gzip, deflate") self.c.setopt(pycurl.HTTPHEADER, ["Accept: */*", - "Accept-Language: en-US,en", - "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7", + "Accept-Language: en-US, en", + "Accept-Charset: ISO-8859-1, utf-8;q=0.7,*;q=0.7", "Connection: keep-alive", "Keep-Alive: 300", "Expect:"]) @@ -167,7 +156,7 @@ class HTTPRequest(): self.c.setopt(pycurl.POSTFIELDS, post) else: - post = [(x, y.encode('utf8') if type(y) == unicode else y ) for x, y in post.iteritems()] + post = [(x, encode(y)) for x, y in post.iteritems()] self.c.setopt(pycurl.HTTPPOST, post) else: self.c.setopt(pycurl.POST, 0) @@ -181,7 +170,7 @@ class HTTPRequest(): self.getCookies() - def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False): + def load(self, url, get={}, post={}, referer=True, cookies=True, just_header=False, multipart=False, decode=False, follow_location=True, save_cookies=True): """ load and returns a given page """ self.setRequestContext(url, get, post, referer, cookies, multipart) @@ -190,24 +179,32 @@ class HTTPRequest(): self.c.setopt(pycurl.HTTPHEADER, self.headers) - if just_header: + if post: + self.c.setopt(pycurl.POST, 1) + else: + self.c.setopt(pycurl.HTTPGET, 1) + + if not follow_location: self.c.setopt(pycurl.FOLLOWLOCATION, 0) + + if just_header: self.c.setopt(pycurl.NOBODY, 1) - self.c.perform() - rep = self.header + self.c.perform() + rep = self.header if just_header else self.getResponse() + + if not follow_location: self.c.setopt(pycurl.FOLLOWLOCATION, 1) - self.c.setopt(pycurl.NOBODY, 0) - else: - self.c.perform() - rep = self.getResponse() + if just_header: + self.c.setopt(pycurl.NOBODY, 0) self.c.setopt(pycurl.POSTFIELDS, "") self.lastEffectiveURL = self.c.getinfo(pycurl.EFFECTIVE_URL) self.code = self.verifyHeader() - self.addCookies() + if save_cookies: + self.addCookies() if decode: rep = self.decodeResponse(rep) @@ -228,11 +225,13 @@ class HTTPRequest(): def getResponse(self): """ retrieve response from string io """ - if self.rep is None: return "" - value = self.rep.getvalue() - self.rep.close() - self.rep = StringIO() - return value + if self.rep is None: + return "" + else: + value = self.rep.getvalue() + self.rep.close() + self.rep = StringIO() + return value def decodeResponse(self, rep): """ decode with correct encoding, relies on header """ @@ -255,7 +254,7 @@ class HTTPRequest(): #self.log.debug("Decoded %s" % encoding ) if lookup(encoding).name == 'utf-8' and rep.startswith(BOM_UTF8): encoding = 'utf-8-sig' - + decoder = getincrementaldecoder(encoding)("replace") rep = decoder.decode(rep, True) @@ -263,6 +262,7 @@ class HTTPRequest(): except LookupError: self.log.debug("No Decoder foung for %s" % encoding) + except Exception: self.log.debug("Error when decoding string from %s." % encoding) @@ -272,13 +272,15 @@ class HTTPRequest(): """ writes response """ if self.rep.tell() > 1000000 or self.abort: rep = self.getResponse() - if self.abort: raise Abort() - f = open("response.dump", "wb") - f.write(rep) - f.close() - raise Exception("Loaded Url exceeded limit") - self.rep.write(buf) + if self.abort: + raise Abort + + with open("response.dump", "wb") as f: + f.write(rep) + raise Fail("Loaded url exceeded size limit") + else: + self.rep.write(buf) def writeHeader(self, buf): """ writes header """ @@ -298,9 +300,3 @@ class HTTPRequest(): if hasattr(self, "c"): self.c.close() del self.c - -if __name__ == "__main__": - url = "http://pyload.org" - c = HTTPRequest() - print c.load(url) - diff --git a/module/network/JsEngine.py b/module/network/JsEngine.py new file mode 100644 index 000000000..2e98fa37d --- /dev/null +++ b/module/network/JsEngine.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- +# @author: vuolter + +import subprocess +import sys + +from os import path +from urllib import quote + +from pyload.utils import encode, decode, uniqify + + +class JsEngine(object): + """ JS Engine superclass """ + + def __init__(self, core, engine=None): + self.core = core + self.engine = None #: Engine Instance + + if not engine: + engine = self.core.config.get("general", "jsengine") + + if engine != "auto" and self.set(engine) is False: + engine = "auto" + self.core.log.warning("JS Engine set to \"auto\" for safely") + + if engine == "auto": + for E in self.find(): + if self.set(E) is True: + break + else: + self.core.log.error("No JS Engine available") + + + @classmethod + def find(cls): + """ Check if there is any engine available """ + return [E for E in ENGINES if E.find()] + + + def get(self, engine=None): + """ Convert engine name (string) to relative JSE class (AbstractEngine extended) """ + if not engine: + JSE = self.engine + + elif isinstance(engine, basestring): + engine_name = engine.lower() + for E in ENGINES: + if E.__name == engine_name: #: doesn't check if E(NGINE) is available, just convert string to class + JSE = E + break + else: + raise ValueError("JSE") + + elif issubclass(engine, AbstractEngine): + JSE = engine + + else: + raise TypeError("engine") + + return JSE + + + def set(self, engine): + """ Set engine name (string) or JSE class (AbstractEngine extended) as default engine """ + if isinstance(engine, basestring): + return self.set(self.get(engine)) + + elif issubclass(engine, AbstractEngine) and engine.find(): + self.engine = engine + return True + + else: + return False + + + def eval(self, script, engine=None): #: engine can be a jse name """string""" or an AbstractEngine """class""" + JSE = self.get(engine) + + if not JSE: + raise TypeError("engine") + + script = encode(script) + + out, err = JSE.eval(script) + + results = [out] + + if self.core.config.get("general", "debug"): + if err: + self.core.log.debug(JSE.__name + ":", err) + + engines = self.find() + engines.remove(JSE) + for E in engines: + out, err = E.eval(script) + res = err or out + self.core.log.debug(E.__name + ":", res) + results.append(res) + + if len(results) > 1 and len(uniqify(results)) > 1: + self.core.log.warning("JS output of two or more engines mismatch") + + return results[0] + + +class AbstractEngine(object): + """ JSE base class """ + + __name = "" + + + def __init__(self): + self.setup() + self.available = self.find() + + + def setup(self): + pass + + + @classmethod + def find(cls): + """ Check if the engine is available """ + try: + __import__(cls.__name) + except Exception: + try: + out, err = cls().eval("print(23+19)") + except Exception: + res = False + else: + res = out == "42" + else: + res = True + + return res + + + def _eval(args): + if not self.available: + return None, "JS Engine \"%s\" not found" % self.__name + + try: + p = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=-1) + return map(lambda x: x.strip(), p.communicate()) + except Exception, e: + return None, e + + + def eval(script): + raise NotImplementedError + + +class Pyv8Engine(AbstractEngine): + + __name = "pyv8" + + + def eval(self, script): + if not self.available: + return None, "JS Engine \"%s\" not found" % self.__name + + try: + rt = PyV8.JSContext() + rt.enter() + res = rt.eval(script), None #@TODO: parse stderr + except Exception, e: + res = None, e + + return res + + +class CommonEngine(AbstractEngine): + + __name = "js" + + + def setup(self): + subprocess.Popen(["js", "-v"], bufsize=-1).communicate() + + + def eval(self, script): + script = "print(eval(unescape('%s')))" % quote(script) + args = ["js", "-e", script] + return self._eval(args) + + +class NodeEngine(AbstractEngine): + + __name = "nodejs" + + + def setup(self): + subprocess.Popen(["node", "-v"], bufsize=-1).communicate() + + + def eval(self, script): + script = "console.log(eval(unescape('%s')))" % quote(script) + args = ["node", "-e", script] + return self._eval(args) + + +class RhinoEngine(AbstractEngine): + + __name = "rhino" + + + def setup(self): + jspath = [ + "/usr/share/java*/js.jar", + "js.jar", + path.join(pypath, "js.jar") + ] + for p in jspath: + if path.exists(p): + self.path = p + break + else: + self.path = "" + + + def eval(self, script): + script = "print(eval(unescape('%s')))" % quote(script) + args = ["java", "-cp", self.path, "org.mozilla.javascript.tools.shell.Main", "-e", script] + res = decode(self._eval(args)) + try: + return res.encode("ISO-8859-1") + finally: + return res + + +class JscEngine(AbstractEngine): + + __name = "javascriptcore" + + + def setup(self): + jspath = "/System/Library/Frameworks/JavaScriptCore.framework/Resources/jsc" + self.path = jspath if path.exists(jspath) else "" + + + def eval(self, script): + script = "print(eval(unescape('%s')))" % quote(script) + args = [self.path, "-e", script] + return self._eval(args) + + +#@NOTE: Priority ordered +ENGINES = [CommonEngine, Pyv8Engine, NodeEngine, RhinoEngine] + +if sys.platform == "darwin": + ENGINES.insert(JscEngine) diff --git a/module/network/RequestFactory.py b/module/network/RequestFactory.py index 5b1528281..579eafea7 100644 --- a/module/network/RequestFactory.py +++ b/module/network/RequestFactory.py @@ -1,32 +1,16 @@ # -*- coding: utf-8 -*- - -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: mkaay, RaNaN -""" +# @author: RaNaN, mkaay from threading import Lock -from Browser import Browser -from Bucket import Bucket -from HTTPRequest import HTTPRequest -from CookieJar import CookieJar +from pyload.network.Browser import Browser +from pyload.network.Bucket import Bucket +from pyload.network.HTTPRequest import HTTPRequest +from pyload.network.CookieJar import CookieJar -from XDCCRequest import XDCCRequest +from pyload.network.XDCCRequest import XDCCRequest -class RequestFactory(): +class RequestFactory(object): def __init__(self, core): self.lock = Lock() self.core = core @@ -62,12 +46,23 @@ class RequestFactory(): def getURL(self, *args, **kwargs): """ see HTTPRequest for argument list """ - h = HTTPRequest(None, self.getOptions()) + cj = None + + if 'cookies' in kwargs: + if isinstance(kwargs['cookies'], CookieJar): + cj = kwargs['cookies'] + elif isinstance(kwargs['cookies'], list): + cj = CookieJar(None) + for cookie in kwargs['cookies']: + if isinstance(cookie, tuple) and len(cookie) == 3: + cj.setCookie(*cookie) + + h = HTTPRequest(cj, self.getOptions()) try: rep = h.load(*args, **kwargs) finally: h.close() - + return rep def getCookieJar(self, pluginName, account=None): diff --git a/module/network/XDCCRequest.py b/module/network/XDCCRequest.py index f03798c17..c49f418c4 100644 --- a/module/network/XDCCRequest.py +++ b/module/network/XDCCRequest.py @@ -1,21 +1,5 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: jeix -""" +# @author: jeix import socket import re @@ -28,22 +12,21 @@ from time import time import struct from select import select -from module.plugins.Plugin import Abort +from pyload.plugin.Plugin import Abort -class XDCCRequest(): +class XDCCRequest(object): def __init__(self, timeout=30, proxies={}): - + self.proxies = proxies self.timeout = timeout - + self.filesize = 0 self.recv = 0 self.speed = 0 - + self.abort = False - def createSocket(self): # proxytype = None # proxy = None @@ -60,78 +43,77 @@ class XDCCRequest(): # else: # sock = socket.socket() # return sock - + return socket.socket() - - def download(self, ip, port, filename, irc, progressNotify=None): + + def download(self, ip, port, filename, irc, progress=None): ircbuffer = "" lastUpdate = time() cumRecvLen = 0 - + dccsock = self.createSocket() - + dccsock.settimeout(self.timeout) dccsock.connect((ip, port)) - + if exists(filename): i = 0 nameParts = filename.rpartition(".") while True: newfilename = "%s-%d%s%s" % (nameParts[0], i, nameParts[1], nameParts[2]) i += 1 - + if not exists(newfilename): filename = newfilename break - + fh = open(filename, "wb") - + # recv loop for dcc socket while True: if self.abort: dccsock.close() fh.close() remove(filename) - raise Abort() - + raise Abort + self._keepAlive(irc, ircbuffer) - + data = dccsock.recv(4096) dataLen = len(data) self.recv += dataLen - + cumRecvLen += dataLen - + now = time() timespan = now - lastUpdate - if timespan > 1: + if timespan > 1: self.speed = cumRecvLen / timespan cumRecvLen = 0 lastUpdate = now - - if progressNotify: - progressNotify(self.percent) - - + + if progress: + progress(self.percent) + if not data: break - + fh.write(data) - + # acknowledge data by sending number of recceived bytes dccsock.send(struct.pack('!I', self.recv)) - + dccsock.close() fh.close() - + return filename - - def _keepAlive(self, sock, readbuffer): + + def _keepAlive(self, sock, *readbuffer): fdset = select([sock], [], [], 0) if sock not in fdset[0]: return - + readbuffer += sock.recv(1024) temp = readbuffer.split("\n") readbuffer = temp.pop() @@ -144,7 +126,7 @@ class XDCCRequest(): def abortDownloads(self): self.abort = True - + @property def size(self): return self.filesize diff --git a/module/network/__init__.py b/module/network/__init__.py index 8b1378917..40a96afc6 100644 --- a/module/network/__init__.py +++ b/module/network/__init__.py @@ -1 +1 @@ - +# -*- coding: utf-8 -*- |