diff options
Diffstat (limited to 'module')
| -rw-r--r-- | module/download_thread.py | 34 | ||||
| -rwxr-xr-x | module/network/Request.py | 11 | ||||
| -rw-r--r-- | module/thread_list.py | 77 |
3 files changed, 70 insertions, 52 deletions
diff --git a/module/download_thread.py b/module/download_thread.py index cced6508a..80a1d4c30 100644 --- a/module/download_thread.py +++ b/module/download_thread.py @@ -19,8 +19,10 @@ ### import threading +import random from time import time, sleep + class Status(object): """ Saves all status information """ @@ -94,31 +96,45 @@ class Download_Thread(threading.Thread): def download(self, pyfile): status = pyfile.status pyfile.prepareDownload() - + print "dl prepared", status.filename + + rnd = random.randint(0,2) + if rnd == 0: + print status.filename, "want reconnect" + status.want_reconnect = True + status.waituntil = time() + 60 + else: + status.waituntil = 0 + status.want_reconnect = False + print status.filename, "doesnt want reconnect" + + if not status.exists: raise "FileDontExists" #i know its deprecated, who cares^^ if status.want_reconnect: - reconnect = self.parent.init_reconnect(pyfile) + reconnect = self.parent.init_reconnect() if reconnect: status.type = "reconnected" + status.want_reconnect = False return False + status.type = "waiting" + while (time() < status.waituntil): if status.want_reconnect and self.parent.reconnecting: status.type = "reconnected" + status.want_reconnect = False return False - status.type = "waiting" sleep(1) status.want_reconnect = False - try: - status.type = "downloading" + + status.type = "downloading" - pyfile.plugin.proceed(status.url, pyfile.download_folder + "/" + status.filename) - status.type = "finished" - except: - status.type = "failed" + pyfile.plugin.proceed(status.url, pyfile.download_folder + "/" + status.filename) + + status.type = "finished" #startet downloader #urllib.urlretrieve(status.url, pyfile.download_folder + "/" + status.filename, status) diff --git a/module/network/Request.py b/module/network/Request.py index 196e350c9..ce6dc6c8c 100755 --- a/module/network/Request.py +++ b/module/network/Request.py @@ -90,7 +90,7 @@ class Request: #def download(url, filename, reporthook = None, data = None): #default von urlretrieve auch None? # return self.downloader.urlretrieve(url, filename, reporthook, data) - def download(self, url, filename, post = {}): + def download(self, url, filename, post={}): if post: post = urllib.urlencode(post) @@ -100,14 +100,13 @@ class Request: if not self.dl: self.dl = True file = open(filename, 'wb') - req = urllib2.Request(url, post) - - conn = self.downloader.open(req) + req = urllib2.Request(url) + conn = self.downloader.open(req, post) self.dl_size = int(conn.headers["content-length"]) self.dl_arrived = 0 self.dl_time = time.time() - for chunk in conn: - self.dl_arrived += len(chunk) + for chunk in conn: + self.dl_arrived += len(chunk) file.write(chunk) file.close() self.dl = False diff --git a/module/thread_list.py b/module/thread_list.py index eb852a37b..874317dc6 100644 --- a/module/thread_list.py +++ b/module/thread_list.py @@ -30,7 +30,7 @@ class Thread_List(object): def __init__(self, parent): self.parent = parent self.threads = [] - self.max_threads = 3 + self.max_threads = 2 self.py_load_files = [] # files in queque self.f_relation = [0, 0] self.lock = RLock() @@ -64,19 +64,21 @@ class Thread_List(object): def get_job(self): # return job if suitable, otherwise send thread idle - self.lock.acquire() - - if not self.py_load_files: - return False + + + if self.reconnecting: + return None + print "get new job" self.init_reconnect() - - if self.reconnecting: - return None + time.sleep(2) if self.pause: return None + + self.lock.acquire() + pyfile = None for i in range(len(self.py_load_files)): if not self.py_load_files[i].modul.__name__ in self.occ_plugins: @@ -96,6 +98,7 @@ class Thread_List(object): def job_finished(self, pyfile): self.lock.acquire() + print pyfile.status.filename, "finished" if not pyfile.plugin.multi_dl: self.occ_plugins.remove(pyfile.modul.__name__) @@ -110,7 +113,7 @@ class Thread_List(object): self.parent.extend_links(pyfile.plugin.links) if pyfile.status.type == "reconnected": - print "put it back" + print "put it back", pyfile.status.filename self.py_load_files.insert(0, pyfile) self.lock.release() @@ -131,42 +134,42 @@ class Thread_List(object): self.f_relation[1] += 1 self.select_thread() - def init_reconnect(self, pyfile=None): + def init_reconnect(self): + self.lock.acquire() - self.lock.acquire() + if self.check_reconnect(): + + print "time to reconnect" - reconnecting = filter(lambda pyfile: pyfile.status.want_reconnect is True, self.py_downloading) #returns all which want to reconenct + self.reconnecting = True + self.reconnect() - if reconnecting and reconnecting == self.py_downloading: - #if empty -> all want reconnect - self.reconnecting = True - self.parent.logger.info("Reconnecting") - self.reconnect() - - self.py_downloading.remove(pyfile) + self.reconnecting = False + + self.lock.release() - while self.py_downloading: - print "waiting" - time.sleep(1) - - self.py_downloading.append(pyfile) - - self.reconnecting = False - - self.lock.release() - - return True + return False + + def check_reconnect(self): + if not self.py_downloading: + return False - else: - self.lock.release() - return False + i = 0 + for obj in self.py_downloading: + if obj.status.want_reconnect: + i += 1 + + if len(self.py_downloading) == i: + return True + else: + return False def reconnect(self): - - print "reconnected" - return True + print "imagine reconnect" + time.sleep(10) + return True reconn = subprocess.Popen(self.parent.config['reconnectMethod']) reconn.wait() @@ -175,4 +178,4 @@ class Thread_List(object): ip = re.match(".*Current IP Address: (.*)</body>.*", urllib2.urlopen("http://checkip.dyndns.org/").read()).group(1) time.sleep(1) self.parent.logger.info("Reconnected, new IP: " + ip) -
\ No newline at end of file + |