From d9c4deacd40bfc16aa8b0092e27241b6b215817a Mon Sep 17 00:00:00 2001 From: Jeix Date: Sun, 20 Jun 2010 11:01:32 +0200 Subject: New feature: XDCC Support Works for links with the following schema: xdcc://.*?/.*?/#?\d+/? (regexp) Example: xdcc://irc.server.net/botname/packid --- module/RequestFactory.py | 16 ++- module/network/XdccRequest.py | 322 ++++++++++++++++++++++++++++++++++++++++++ module/plugins/Plugin.py | 2 +- module/plugins/hoster/Xdcc.py | 70 +++++++++ 4 files changed, 404 insertions(+), 6 deletions(-) create mode 100644 module/network/XdccRequest.py create mode 100644 module/plugins/hoster/Xdcc.py diff --git a/module/RequestFactory.py b/module/RequestFactory.py index e707ea6cd..410d5ccd5 100644 --- a/module/RequestFactory.py +++ b/module/RequestFactory.py @@ -19,6 +19,7 @@ from threading import Lock from module.network.Request import Request +from module.network.XdccRequest import XdccRequest import pycurl class RequestFactory(): @@ -28,12 +29,17 @@ class RequestFactory(): self.requests = [] self.cookiejars = [] - def getRequest(self, pluginName, account=None): + def getRequest(self, pluginName, account=None, type="HTTP"): self.lock.acquire() - iface = self.core.config["general"]["download_interface"] - req = Request(interface=str(iface)) - cj = self.getCookieJar(pluginName, account) - req.setCookieJar(cj) + if type == "HTTP": + iface = self.core.config["general"]["download_interface"] + req = Request(interface=str(iface)) + cj = self.getCookieJar(pluginName, account) + req.setCookieJar(cj) + + elif type == "XDCC": + req = XdccRequest() + self.requests.append((pluginName, account, req)) self.lock.release() return req diff --git a/module/network/XdccRequest.py b/module/network/XdccRequest.py new file mode 100644 index 000000000..c7c4bc893 --- /dev/null +++ b/module/network/XdccRequest.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: spoob + @author: RaNaN + @author: mkaay + @author: jeix + @version: v0.3.2 +""" + +import time +import socket +from select import select +import re +from os.path import exists +import struct + +class AbortDownload(Exception): + pass + +class IRCError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class XDCCError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class XdccRequest: + def __init__(self): + + self.dl_time = 0 + self.dl_finished = 0 + self.dl_size = 0 + self.dl_arrived = 0 + self.dl = False + + self.abort = False + + self.timeout = 60 + + bufferBase = 1024 + bufferMulti = 4 + self.bufferSize = bufferBase*bufferMulti + self.canContinue = False + self.offset = 0 + + self.dl_speed = 0.0 + self.averageSpeed = 0.0 + self.averageSpeeds = [] + self.averageSpeedTime = 0.0 + self.averageSpeedCount = 0.0 + + self.speedLimitActive = False + self.maxSpeed = 0 + self.isSlow = False + + # change this for connection information + self.debug = False + + def set_timeout(self, timeout): + self.timeout = int(timeout) + + def add_proxy(self, protocol, adress): + # @TODO: pycurl proxy protocoll selection + raise NotImplementedError + + # xdcc://irc.Abjects.net/[XDCC]|Shit/#0004/ + #nick, ident, realname, servers + def download(self, bot, pack, path, nick, ident, realname, host, port=6667): + self.dl_time = time.time() + self.dl = True + + self.chunkSize = 0 + self.chunkRead = 0 + self.subStartTime = 0 + self.maxChunkSize = 0 + + def restLimit(): + subTime = time.time() - self.subStartTime + if subTime <= 1: + if self.speedLimitActive: + return self.maxChunkSize + else: + return -1 + else: + self.updateCurrentSpeed(float(self.chunkRead/1024) / subTime) + + self.subStartTime = time.time() + self.chunkRead = 0 + if self.maxSpeed > 0: + self.maxChunkSize = self.maxSpeed + else: + self.maxChunkSize = 0 + return 0 + + def writefunc(in_chunkSize): + chunkSize = in_chunkSize + while chunkSize > restLimit() > -1: + time.sleep(0.05) + self.maxChunkSize -= chunkSize + self.chunkRead += chunkSize + self.dl_arrived += chunkSize + + + # connect to IRC + sock = socket.socket() + sock.connect((host, port)) + if nick == "pyload": + nick = "pyload-%d" % (time.time() % 1000) # last 3 digits + sock.send("NICK %s\r\n" % nick) + sock.send("USER %s %s bla :%s\r\n" % (ident, host, realname)) + sock.send("JOIN #jeixus\r\n") + sock.send("PRIVMSG %s :xdcc send #%s\r\n" % (bot, pack)) + + # IRC recv loop + readbuffer = "" + while True: + if self.abort: + raise AbortDownload + + if self.dl_time + self.timeout*60 < time.time(): + raise XDCCError("timout, bot did not answer") + + #time.sleep(5) # cool down <- was a bullshit idea + + fdset = select([sock], [], [], 0) + if sock not in fdset[0]: + continue + + readbuffer += sock.recv(1024) + temp = readbuffer.split("\n") + readbuffer = temp.pop() + + for line in temp: + if self.debug: print "*> " + line + line = line.rstrip() + first = line.split() + + if(first[0] == "PING"): + sock.send("PONG %s\r\n" % first[1]) + + if first[0] == "ERROR": + raise IRCError(line) + + msg = line.split(None, 3) + if len(msg) != 4: + continue + + msg = { \ + "origin":msg[0][1:], \ + "action":msg[1], \ + "target":msg[2], \ + "text" :msg[3][1:] \ + } + if not (bot == msg["origin"][0:len(bot)] + and nick == msg["target"][0:len(nick)] + and "PRIVMSG" == msg["action"]): + continue + + m = re.match('\x01DCC SEND (.*?) (.*?) (.*?) (.*?)\x01', msg["text"]) + if m != None: + break + + # kill IRC socket + sock.send("QUIT :byebye\r\n") + + # connect to XDCC Bot + dcc = socket.socket() + ip = socket.inet_ntoa(struct.pack('L', socket.ntohl(int(m.group(2))))) + port = int(m.group(3)) + dcc.connect((ip, port)) + + dcc_packname = m.group(1) + if len(m.groups()) > 3: + self.dl_size = int(m.group(4)) + dcc_packname = self.get_free_name(path + '\\' + dcc_packname) + dcc_fpointer = open(dcc_packname, "wb") + dcc_total = 0 + + # recv loop for dcc socket + while True: + if self.abort: + dcc.close() + dcc_fpointer.close() + raise AbortDownload + + fdset = select([dcc], [], [], 0) + if dcc not in fdset[0]: + continue + + # recv something + recvbytes = dcc.recv(2**14) + + # connection closed and everything received -> reset variables + if len(recvbytes) == 0: + dcc.close() + dcc_fpointer.close() + break + + # status updates, speedmanaging, etc. + writefunc(len(recvbytes)) + + # add response to file + dcc_fpointer.write(recvbytes) + dcc_total += len(recvbytes) + + # acknowledge data by sending number of recceived bytes + dcc.send(struct.pack('!I', dcc_total)) + ######################## + free_name = self.get_free_name(file_name) + rename(file_temp, free_name) + + self.dl = False + self.dl_finished = time.time() + + return free_name + + def updateCurrentSpeed(self, speed): + self.dl_speed = speed + if self.averageSpeedTime + 10 < time.time(): + self.averageSpeeds = [] + self.averageSpeeds.append(self.averageSpeed) + self.averageSpeeds.append(speed) + self.averageSpeed = (speed + self.averageSpeed)/2 + self.averageSpeedTime = time.time() + self.averageSpeedCount = 2 + else: + self.averageSpeeds.append(speed) + self.averageSpeedCount += 1 + allspeed = 0.0 + for s in self.averageSpeeds: + allspeed += s + self.averageSpeed = allspeed / self.averageSpeedCount + + def write_header(self, string): + self.header += string + + def get_rep(self): + value = self.rep.getvalue() + self.rep.close() + self.rep = StringIO() + return value + + def get_header(self): + h = self.header + self.header = "" + return h + + def get_speed(self): + try: + return self.dl_speed + except: + return 0 + + def get_ETA(self): + try: + return (self.dl_size - self.dl_arrived) / (self.dl_arrived / (time.time() - self.dl_time)) + except: + return 0 + + def kB_left(self): + return (self.dl_size - self.dl_arrived) / 1024 + + def progress(self, dl_t, dl_d, up_t, up_d): + if self.abort: + return False + self.dl_arrived = int(dl_d) + self.dl_size = int(dl_t) + + def get_free_name(self, file_name): + file_count = 0 + while exists(file_name): + file_count += 1 + if "." in file_name: + file_split = file_name.split(".") + temp_name = "%s-%i.%s" % (".".join(file_split[:-1]), file_count, file_split[-1]) + else: + temp_name = "%s-%i" % (file_name, file_count) + if not exists(temp_name): + file_name = temp_name + return file_name + + def __del__(self): + self.clean() + + def clean(self): + try: + pass + # self.pycurl.close() + except: + pass + +# def getURL(url): + # """ + # currently used for update check + # """ + # req = Request() + # c = req.load(url) + # req.pycurl.close() + # return c + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/module/plugins/Plugin.py b/module/plugins/Plugin.py index 10cb1d6a0..0e04a86e6 100644 --- a/module/plugins/Plugin.py +++ b/module/plugins/Plugin.py @@ -137,7 +137,7 @@ class Plugin(): self.ocr = captchaClass() def __call__(self): - return self.props['name'] + raise NotImplementedError def check_file(self, local_file): """ diff --git a/module/plugins/hoster/Xdcc.py b/module/plugins/hoster/Xdcc.py new file mode 100644 index 000000000..4c529b3bc --- /dev/null +++ b/module/plugins/hoster/Xdcc.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: jeix +""" + +import logging +from os.path import exists +from os.path import join +from os.path import exists +from os import makedirs +import re +import sys + +from module.plugins.Hoster import Hoster + + +class Xdcc(Hoster): + __name__ = "Xdcc" + __version__ = "0.1" + __pattern__ = r'xdcc://.*?/.*?/#?\d+/?' # xdcc://irc.Abjects.net/[XDCC]|Shit/#0004/ + __type__ = "hoster" + __description__ = """A Plugin that allows you to download from an IRC XDCC bot""" + __author_name__ = ("jeix") + __author_mail__ = ("jeix@hasnomail.com") + + def __init__(self, parent): + self.parent = parent + self.req = parent.core.requestFactory.getRequest(self.__name__, type="XDCC") + self.want_reconnect = False + self.multi_dl = True + self.logger = logging.getLogger("log") + self.pyfile = self.parent + + def prepare(self, thread): + self.pyfile.status.url = self.parent.url + thread.wait(self.parent) + return True + + def proceed(self, url, location): + download_folder = self.parent.core.config['general']['download_folder'] + location = download_folder + if self.pyfile.package.data["package_name"] != (self.parent.core.config['general']['link_file']) and self.parent.core.xmlconfig.get("general", "folder_per_package", False): + self.pyfile.folder = self.pyfile.package.data["package_name"] + location = join(download_folder, self.pyfile.folder.decode(sys.getfilesystemencoding())) + if not exists(location): + makedirs(location) + + m = re.search(r'xdcc://(.*?)/(.*?)/#?(\d+)/?', url) + server = m.group(1) + bot = m.group(2) + pack = m.group(3) + nick = self.parent.core.config['xdcc']['nick'] + ident = self.parent.core.config['xdcc']['ident'] + real = self.parent.core.config['xdcc']['realname'] + + self.pyfile.status.filename = self.req.download(bot, pack, location, nick, ident, real, server) -- cgit v1.2.3