From 641cd8c63f4c3a96a9acf4d46450ab975b9c66cd Mon Sep 17 00:00:00 2001 From: RaNaN Date: Wed, 20 May 2009 11:05:21 +0000 Subject: Neue Request Klasse eingebunden + ein paar Status Funktionen --- Core.py | 13 +++-- Plugins/Plugin.py | 3 +- Plugins/RapidshareCom.py | 7 +-- Plugins/UploadedTo.py | 3 +- Py_Load_File.py | 28 ----------- download_thread.py | 114 ------------------------------------------ module/Py_Load_File.py | 28 +++++++++++ module/download_thread.py | 124 ++++++++++++++++++++++++++++++++++++++++++++++ module/network/Request.py | 77 ++++++++++++++++++---------- module/thread_list.py | 87 ++++++++++++++++++++++++++++++++ thread_list.py | 82 ------------------------------ 11 files changed, 307 insertions(+), 259 deletions(-) delete mode 100644 Py_Load_File.py delete mode 100644 download_thread.py create mode 100644 module/Py_Load_File.py create mode 100644 module/download_thread.py create mode 100644 module/thread_list.py delete mode 100644 thread_list.py diff --git a/Core.py b/Core.py index 4093927f4..1a0e3e1ae 100644 --- a/Core.py +++ b/Core.py @@ -33,9 +33,9 @@ from time import sleep, time import pickle #my imports -from download_thread import Download_Thread -from thread_list import Thread_List -from Py_Load_File import PyLoadFile +from module.download_thread import Download_Thread +from module.thread_list import Thread_List +from module.Py_Load_File import PyLoadFile basicConfig(filename='Logs/faild.txt', format = '%(message)s') @@ -216,16 +216,19 @@ class Core(object): def __new_py_load_file(self, url, plugin): - #plugin_name = plugin.__name__ new_file = PyLoadFile(self, plugin, url) new_file.download_folder = self.download_folder self.thread_list.append_py_load_file(new_file) return True def _test_print_status(self): - if len(self.thread_list.threads)>0: + if len(self.thread_list.py_load_files)>0: + for pyfile in self.thread_list.py_load_files: if pyfile.status.type == 'downloading': + print "Speed" ,pyfile.status.getSpeed() + print "ETA" , pyfile.status.getETA() + try: fn = pyfile.status.filename p = round(float(pyfile.status.downloaded_kb)/pyfile.status.total_kb, 2) diff --git a/Plugins/Plugin.py b/Plugins/Plugin.py index 44ba119e1..971677ec7 100644 --- a/Plugins/Plugin.py +++ b/Plugins/Plugin.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- from time import time - +from module.network.Request import Request class Plugin(): @@ -19,6 +19,7 @@ class Plugin(): pluginProp ['author_email'] = "nn@nn.de" self.pluginProp = pluginProp self.parent = parent + self.req = Request() self.html = None self.time_plus_wait = None #time() + wait in seconds self.want_reconnect = None diff --git a/Plugins/RapidshareCom.py b/Plugins/RapidshareCom.py index 6f81a9549..8adf89e25 100644 --- a/Plugins/RapidshareCom.py +++ b/Plugins/RapidshareCom.py @@ -12,6 +12,7 @@ from time import time class RapidshareCom(Plugin): def __init__(self, parent): + Plugin.__init__(self, parent) self.plugin_name = "Rapidshare.com" self.plugin_pattern = r"http://(?:www.)?rapidshare.com/files/" self.plugin_type = "hoster" @@ -46,7 +47,7 @@ class RapidshareCom(Plugin): """ gets the url from self.parent.url saves html in self.html and parses """ url = self.parent.url - html = urllib2.urlopen(url).read() + html = self.req.load(url) self.html[0] = html self.html_old = time() @@ -58,8 +59,8 @@ class RapidshareCom(Plugin): file_server_url = re.search(r"
. -# -### - -import threading -from time import time, sleep -from copy import copy -import urllib - -class Status(object): - """ Saves all status information - """ - def __init__(self, id): - self.type = None - self.status_queue = None - self.id = id - self.total_kb = 0 - self.downloaded_kb = 0 - self.rate = 0 - self.expected_time = 0 - self.filename = None - self.url = None - self.exists = None - self.waituntil = None - self.want_reconnect = None - - def __call__(self, blocks_read, block_size, total_size): - if self.status_queue == None: - return False - self.start = time() - self.last_status = time() - self.total_kb = total_size / 1024 - self.downloaded_kb = (blocks_read * block_size) / 1024 - elapsed_time = time() - self.start - if elapsed_time != 0: - self.rate = self.downloaded_kb / elapsed_time - if self.rate != 0: - self.expected_time = self.downloaded_kb / self.rate - if self.last_status+0.2 < time(): - self.status_queue.put(copy(self)) - self.last_status = time() - - def set_status_queue(self, queue): - self.status_queue = queue - - -class Download_Thread(threading.Thread): - def __init__(self, parent): - threading.Thread.__init__(self) - self.shutdown = False - self.parent = parent - self.setDaemon(True) - self.start() - - def run(self): - while (not self.shutdown): - if not self.parent.download_queue.empty(): - py_load_file = self.parent.download_queue.get() - self.download(py_load_file) - if self.shutdown: - sleep(1) - self.parent.remove_thread(self) - - def download(self, py_load_file): - pyfile = py_load_file - status = pyfile.status - pyfile.prepareDownload() - - if not status.exists: - self.shutdown = True - return False - - if status.want_reconnect: - print "handle reconnect" - self.shutdown = True - return False - - while (time() < status.waituntil): - status.type = "waiting" - sleep(1) #eventuell auf genaue zeit warten - - #missing wenn datei nicht auf server vorhanden - #if type=="check": - #return params - #if type in 'missing': - #self.status = "missing" - #print "Datei auf Server nicht vorhanden: " + params - ##im logger eintragen das datei auf server nicht vorhanden ist - #warning("Datei auf Server nicht voblocks_readrhanden: " + url) - #if type in 'wait': - # status.type = "waiting" - # print params - # sleep(params+1) - print "going to download" - status.type = "downloading" - #startet downloader - urllib.urlretrieve(status.url, pyfile.download_folder + "/" + status.filename, status) - self.shutdown = True diff --git a/module/Py_Load_File.py b/module/Py_Load_File.py new file mode 100644 index 000000000..4a15cd990 --- /dev/null +++ b/module/Py_Load_File.py @@ -0,0 +1,28 @@ +from download_thread import Status + +class PyLoadFile: + """ represents the url or file + """ + def __init__(self, parent, plugin, url): + self.parent = parent + self.id = None + pluginClass = getattr(plugin, plugin.__name__) + self.plugin = pluginClass(self) + self.url = url + self.filename = "filename" + self.download_folder = "" + self.status = Status(self) + + + def _get_my_plugin(): + plugins = parent.get_avail_plugins() + + + def prepareDownload(self): + self.status.exists = self.plugin.file_exists() + if self.status.exists: + self.status.filename = self.plugin.get_file_name() + self.status.waituntil = self.plugin.time_plus_wait + self.status.url = self.plugin.get_file_url() + self.status.want_reconnect = self.plugin.want_reconnect + diff --git a/module/download_thread.py b/module/download_thread.py new file mode 100644 index 000000000..f6f43a295 --- /dev/null +++ b/module/download_thread.py @@ -0,0 +1,124 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2009 sp00b, sebnapi +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +### + +import threading +from time import time, sleep +from copy import copy +import urllib + + +class Status(object): + """ Saves all status information + """ + def __init__(self, pyfile): + self.pyfile = pyfile + self.type = None + self.status_queue = None + self.total_kb = 0 + self.downloaded_kb = 0 + self.rate = 0 + self.expected_time = 0 + self.filename = None + self.url = None + self.exists = None + self.waituntil = None + self.want_reconnect = None + + def __call__(self, blocks_read, block_size, total_size): + if self.status_queue == None: + return False + self.start = time() + self.last_status = time() + self.total_kb = total_size / 1024 + self.downloaded_kb = (blocks_read * block_size) / 1024 + elapsed_time = time() - self.start + if elapsed_time != 0: + self.rate = self.downloaded_kb / elapsed_time + if self.rate != 0: + self.expected_time = self.downloaded_kb / self.rate + if self.last_status+0.2 < time(): + self.status_queue.put(copy(self)) + self.last_status = time() + + def set_status_queue(self, queue): + self.status_queue = queue + + def getETA(): + return self.pyfile.plugin.req.getETA() + def getSpeed(): + return self.pyfile.plugin.req.getSpeed() + def kBleft(): + return self.pyfile.plugins.req.kBleft() + + +class Download_Thread(threading.Thread): + def __init__(self, parent): + threading.Thread.__init__(self) + self.shutdown = False + self.parent = parent + self.setDaemon(True) + self.loadedPyFile = None + + self.start() + + def run(self): + while (not self.shutdown): + if not self.parent.download_queue.empty(): + self.loadedPyFile = self.parent.getJob() + self.download(self.loadedPyFile) + + if self.shutdown: + sleep(1) + self.parent.remove_thread(self) + + def download(self, py_load_file): + pyfile = py_load_file + status = pyfile.status + pyfile.prepareDownload() + + if not status.exists: + return False + + if status.want_reconnect: + print "handle reconnect" + return False + + while (time() < status.waituntil): + status.type = "waiting" + sleep(1) #eventuell auf genaue zeit warten + + #missing wenn datei nicht auf server vorhanden + #if type=="check": + #return params + #if type in 'missing': + #self.status = "missing" + #print "Datei auf Server nicht vorhanden: " + params + ##im logger eintragen das datei auf server nicht vorhanden ist + #warning("Datei auf Server nicht voblocks_readrhanden: " + url) + + print "going to download" + status.type = "downloading" + print status.url , status.filename + + pyfile.plugin.req.download(status.url, pyfile.download_folder + "/" + status.filename) + status.type = "finished" + #startet downloader + #urllib.urlretrieve(status.url, pyfile.download_folder + "/" + status.filename, status) + #self.shutdown = True diff --git a/module/network/Request.py b/module/network/Request.py index 3325d085d..07b0129d2 100755 --- a/module/network/Request.py +++ b/module/network/Request.py @@ -7,6 +7,8 @@ import urllib import urllib2 import cookielib import Keepalive +import base64 +import time from Keepalive import HTTPHandler from cStringIO import StringIO @@ -21,23 +23,19 @@ from gzip import GzipFile retrieveUrl returns response as string """ -class Downloader(urllib.FancyURLopener): - version = "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8" - - - class Request: def __init__(self): + + self.dl_time = 0 + self.dl_size = 0 + self.dl_arrived = 0 + self.dl = False self.lastURL = None - #self.cookiefile = 'cookies.lwp' self.cj = cookielib.CookieJar() - -# if os.path.isfile(self.cookiefile): - # self.cj.load(self.cookiefile) - - self.handler = HTTPHandler() - self.opener = urllib2.build_opener(self.handler, urllib2.HTTPCookieProcessor(self.cj)) + handler = HTTPHandler() + self.opener = urllib2.build_opener(handler, urllib2.HTTPCookieProcessor(self.cj)) + self.downloader = urllib2.build_opener() #self.opener.add_handler() self.opener.addheaders = [ @@ -47,11 +45,15 @@ class Request: ("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7"), ("Connection","keep-alive"), ("Keep-Alive","300")] - - self.downloader = Downloader() + + self.downloader.addheaders = [ + ("User-Agent","Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8"), + ("Accept-Encoding","gzip,deflate"), + ("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + ("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7")] - def retrieveURL(self,url, get = {}, post = {}, ref = True): + def load(self, url, get = {}, post = {}, ref = True): if post: post = urllib.urlencode(post) @@ -72,11 +74,7 @@ class Request: rep = self.opener.open(req) output = rep.read() - - print rep.headers - - self.cj.extract_cookies(rep, req) - + if rep.headers.has_key("content-encoding") : if rep.headers["content-encoding"] == "gzip" : output = GzipFile('','r',0,StringIO(output)).read() @@ -86,13 +84,42 @@ class Request: return output def addAuth(self, user, pw): - auth_handler = urllib2.HTTPBasicAuthHandler() - auth_handler.add_password(user, passwd= pw) - self.opener.add_handler(auth_handler) + self.downloader.addheaders.append(['Authorization','Basic ' + base64.encodestring(user + ':' + pw)[:-1]]) + + + #def download(url, filename, reporthook = None, data = None): #default von urlretrieve auch None? + # return self.downloader.urlretrieve(url, filename, reporthook, data) + + def download(self, url, filename): + if not self.dl: + self.dl = True + file = open(filename, 'wb') + req = urllib2.Request(url) + conn = self.downloader.open(req) + self.dl_size = int(conn.headers["content-length"]) + self.dl_arrived = 0 + self.dl_time = time.time() + for chunk in conn: + self.dl_arrived += len(chunk) + file.write(chunk) + file.close() + self.dl = False + return True + + def getSpeed(self): + try: + return (self.dl_arrived / (time.time() - self.dl_time)) / 1024 + except: + return "No Download" - def download(url, filename, reporthook = None, data = None): #default von urlretrieve auch None? - return self.downloader.urlretrieve(url, filename, reporthook, data) + def getETA(self): + try: + return (self.dl_size - self.dl_arrived) / (self.dl_arrived / (time.time() - self.dl_time)) + except: + return "No Download" + def kBleft(self): + return (self.dl_size - self.dl_arrived) / 1024 if __name__ == "__main__" : import doctest diff --git a/module/thread_list.py b/module/thread_list.py new file mode 100644 index 000000000..2a5dbe6f6 --- /dev/null +++ b/module/thread_list.py @@ -0,0 +1,87 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +#Copyright (C) 2009 sp00b, sebnapi +# +#This program is free software; you can redistribute it and/or modify +#it under the terms of the GNU General Public License as published by +#the Free Software Foundation; either version 3 of the License, +#or (at your option) any later version. +# +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#See the GNU General Public License for more details. +# +#You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +### +#python +from Queue import Queue + +#my +from download_thread import Download_Thread + +class Thread_List(object): + def __init__(self, parent): + self.parent = parent + self.threads = [] + self.max_threads = 3 + self.py_load_files = [] + self.download_queue = Queue() + self.status_queue = Queue() + self.f_relation = [0,0] + + def create_thread(self): + """ creates thread for Py_Load_File and append thread to self.threads + """ + if self.py_load_files: + thread = Download_Thread(self) + self.threads.append(thread) + return True + + def get_loaded_urls(self): + loaded_urls = [] + for file in self.py_load_files: + loaded_urls.append(file.url) + return loaded_urls + + def remove_thread(self, thread): + self.threads.remove(thread) + + def status(self): + if not self.status_queue.empty(): + while not self.status_queue.empty(): + status = self.status_queue.get() + self.py_load_files[status.id].status = status + + def getJob(self): + # nur wenn auch geladen werden soll, ansonsten thread in leerlauf schicken + if True: + return self.download_queue.get() + + def extend_py_load_files(self): + pass + + def select_thread(self): + """ select a thread + """ + if len(self.threads) < self.max_threads: + self.create_thread() + + def append_py_load_file(self, py_load_file): + py_load_file.id = len(self.py_load_files) + self.py_load_files.append(py_load_file) + self.download_queue.put(py_load_file) + self.f_relation[1] += 1 + self.select_thread() + + def reconnect(): + reconn = subprocess.Popen(reconnectMethod) + reconn.wait() + ip = re.match(".*Current IP Address: (.*).*", urllib2.urlopen("http://checkip.dyndns.org/").read()).group(1) #versuchen neue ip aus zu lesen + while ip == "": #solange versuch bis neue ip ausgelesen + ip = re.match(".*Current IP Address: (.*).*", urllib2.urlopen("http://checkip.dyndns.org/").read()).group(1) + time.sleep(1) + #print "Neue IP: " + ip diff --git a/thread_list.py b/thread_list.py deleted file mode 100644 index 0dd61ec7a..000000000 --- a/thread_list.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -#Copyright (C) 2009 sp00b, sebnapi -# -#This program is free software; you can redistribute it and/or modify -#it under the terms of the GNU General Public License as published by -#the Free Software Foundation; either version 3 of the License, -#or (at your option) any later version. -# -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#See the GNU General Public License for more details. -# -#You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -### -#python -from Queue import Queue - -#my -from download_thread import Download_Thread - -class Thread_List(object): - def __init__(self, parent): - self.parent = parent - self.threads = [] - self.max_threads = 3 - self.py_load_files = [] - self.download_queue = Queue() - self.status_queue = Queue() - self.f_relation = [0,0] - - def create_thread(self): - """ creates thread for Py_Load_File and append thread to self.threads - """ - if self.py_load_files: - thread = Download_Thread(self) - self.threads.append(thread) - return True - - def get_loaded_urls(self): - loaded_urls = [] - for file in self.py_load_files: - loaded_urls.append(file.url) - return loaded_urls - - def remove_thread(self, thread): - self.threads.remove(thread) - - def status(self): - if not self.status_queue.empty(): - while not self.status_queue.empty(): - status = self.status_queue.get() - self.py_load_files[status.id].status = status - - def extend_py_load_files(self): - pass - - def select_thread(self): - """ select a thread - """ - if len(self.threads) < self.max_threads: - self.create_thread() - - def append_py_load_file(self, py_load_file): - py_load_file.id = len(self.py_load_files) - self.py_load_files.append(py_load_file) - self.download_queue.put(py_load_file) - self.f_relation[1] += 1 - self.select_thread() - - def reconnect(): - reconn = subprocess.Popen(reconnectMethod) - reconn.wait() - ip = re.match(".*Current IP Address: (.*).*", urllib2.urlopen("http://checkip.dyndns.org/").read()).group(1) #versuchen neue ip aus zu lesen - while ip == "": #solange versuch bis neue ip ausgelesen - ip = re.match(".*Current IP Address: (.*).*", urllib2.urlopen("http://checkip.dyndns.org/").read()).group(1) - time.sleep(1) - #print "Neue IP: " + ip -- cgit v1.2.3