Neue Request Klasse eingebunden + ein paar Status Funktionen

8 files changed, 96 insertions, 48 deletions

diff --git a/Core.py b/Core.py
index 4093927f4..1a0e3e1ae 100644
--- a/Core.py
+++ b/Core.py
@@ -33,9 +33,9 @@ from time import sleep, time
 import pickle
 
 #my imports
-from download_thread import Download_Thread
-from thread_list import Thread_List
-from Py_Load_File import PyLoadFile
+from module.download_thread import Download_Thread
+from module.thread_list import Thread_List
+from module.Py_Load_File import PyLoadFile
 
 basicConfig(filename='Logs/faild.txt', format = '%(message)s')
 
@@ -216,16 +216,19 @@ class Core(object):
             
             
     def __new_py_load_file(self, url, plugin):
-        #plugin_name = plugin.__name__
         new_file = PyLoadFile(self, plugin, url)
         new_file.download_folder = self.download_folder
         self.thread_list.append_py_load_file(new_file)
         return True
     
     def _test_print_status(self):
-        if len(self.thread_list.threads)>0:
+        if len(self.thread_list.py_load_files)>0:
+                
             for pyfile in self.thread_list.py_load_files:
                 if pyfile.status.type == 'downloading':
+                    print "Speed" ,pyfile.status.getSpeed()
+                    print "ETA" , pyfile.status.getETA()
+
                     try:
                         fn = pyfile.status.filename
                         p = round(float(pyfile.status.downloaded_kb)/pyfile.status.total_kb, 2)
diff --git a/Plugins/Plugin.py b/Plugins/Plugin.py
index 44ba119e1..971677ec7 100644
--- a/Plugins/Plugin.py
+++ b/Plugins/Plugin.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 from time import time
-
+from module.network.Request import Request
 
 class Plugin():
     
@@ -19,6 +19,7 @@ class Plugin():
         pluginProp ['author_email'] = "nn@nn.de"
         self.pluginProp = pluginProp 
         self.parent = parent
+        self.req = Request()
         self.html = None
         self.time_plus_wait = None #time() + wait in seconds
         self.want_reconnect = None
diff --git a/Plugins/RapidshareCom.py b/Plugins/RapidshareCom.py
index 6f81a9549..8adf89e25 100644
--- a/Plugins/RapidshareCom.py
+++ b/Plugins/RapidshareCom.py
@@ -12,6 +12,7 @@ from time import time
 class RapidshareCom(Plugin):
     
     def __init__(self, parent):
+        Plugin.__init__(self, parent)
         self.plugin_name = "Rapidshare.com"
         self.plugin_pattern = r"http://(?:www.)?rapidshare.com/files/"
         self.plugin_type = "hoster"
@@ -46,7 +47,7 @@ class RapidshareCom(Plugin):
         """ gets the url from self.parent.url saves html in self.html and parses
         """ 
         url = self.parent.url
-        html = urllib2.urlopen(url).read()
+        html = self.req.load(url)
         self.html[0] = html
         self.html_old = time()
 
@@ -58,8 +59,8 @@ class RapidshareCom(Plugin):
         
         file_server_url = re.search(r"<form action=\"(.*?)\"", self.html[0]).group(1)
         #free user
-        free_user_encode = urllib.urlencode({"dl.start" : "Free"})
-        self.html[1] = urllib2.urlopen(file_server_url, free_user_encode).read()
+        #free_user_encode = urllib.urlencode({"dl.start" : "Free"})
+        self.html[1] = self.req.load(file_server_url, None,{"dl.start" : "Free"})
         self.html_old = time()
         self.get_wait_time()
         
diff --git a/Plugins/UploadedTo.py b/Plugins/UploadedTo.py
index fe2c3ec37..0d00c8aa6 100644
--- a/Plugins/UploadedTo.py
+++ b/Plugins/UploadedTo.py
@@ -8,6 +8,7 @@ from Plugin import Plugin
 class UploadedTo(Plugin):
     
     def __init__(self, parent):
+        Plugin.__init__(self, parent)
         self.plugin_name = "Uploaded.to"
         self.plugin_pattern = r"http://(www\.)?uploaded.to/"
         self.plugin_type = "hoster"
@@ -37,7 +38,7 @@ class UploadedTo(Plugin):
         
     def download_html(self):
         url = self.parent.url
-        self.html = urllib2.urlopen(url).read()
+        self.html = req.load(url)
 
         try:
             wait_minutes = re.search(r"Or wait (\d+) minutes", self.html).group(1)
diff --git a/Py_Load_File.py b/module/Py_Load_File.py
index 157d51bfd..4a15cd990 100644
--- a/Py_Load_File.py
+++ b/module/Py_Load_File.py
@@ -11,7 +11,7 @@ class PyLoadFile:
         self.url = url
 	self.filename = "filename"
         self.download_folder = ""
-        self.status = Status(self.id)
+        self.status = Status(self)
 
     
     def _get_my_plugin():
diff --git a/download_thread.py b/module/download_thread.py
index 672f15682..f6f43a295 100644
--- a/download_thread.py
+++ b/module/download_thread.py
@@ -23,13 +23,14 @@ from time import time, sleep
 from copy import copy
 import urllib
 
+
 class Status(object):
     """ Saves all status information
     """
-    def __init__(self, id):
+    def __init__(self, pyfile):
+        self.pyfile = pyfile
         self.type = None
         self.status_queue = None
-        self.id = id
         self.total_kb = 0
         self.downloaded_kb = 0
         self.rate = 0
@@ -58,6 +59,13 @@ class Status(object):
     
     def set_status_queue(self, queue):
         self.status_queue = queue
+
+    def getETA():
+        return self.pyfile.plugin.req.getETA()
+    def getSpeed():
+        return self.pyfile.plugin.req.getSpeed()
+    def kBleft():
+        return self.pyfile.plugins.req.kBleft()
             
             
 class Download_Thread(threading.Thread):
@@ -66,13 +74,16 @@ class Download_Thread(threading.Thread):
         self.shutdown = False
         self.parent = parent
         self.setDaemon(True)
+        self.loadedPyFile = None
+        
         self.start()
         
     def run(self):
         while (not self.shutdown):
             if not self.parent.download_queue.empty():
-                py_load_file = self.parent.download_queue.get()
-            self.download(py_load_file)
+                self.loadedPyFile = self.parent.getJob()
+                self.download(self.loadedPyFile)
+                
         if self.shutdown:
             sleep(1)
             self.parent.remove_thread(self)
@@ -83,12 +94,10 @@ class Download_Thread(threading.Thread):
         pyfile.prepareDownload()
 
 	if not status.exists:
-            self.shutdown = True
             return False
             
         if status.want_reconnect:
             print "handle reconnect"
-            self.shutdown = True
             return False
         
     	while (time() < status.waituntil):
@@ -103,12 +112,13 @@ class Download_Thread(threading.Thread):
             #print "Datei auf Server nicht vorhanden: " + params
             ##im logger eintragen das datei auf server nicht vorhanden ist
             #warning("Datei auf Server nicht voblocks_readrhanden: " + url)
-	#if type in 'wait':
-	 #   status.type = "waiting"
-	  #  print params
-	   # sleep(params+1)
+
         print "going to download"
         status.type = "downloading"
+        print status.url , status.filename
+
+        pyfile.plugin.req.download(status.url, pyfile.download_folder + "/" + status.filename)
+        status.type = "finished"
         #startet downloader
-        urllib.urlretrieve(status.url, pyfile.download_folder + "/" + status.filename, status)
-        self.shutdown = True
+        #urllib.urlretrieve(status.url, pyfile.download_folder + "/" + status.filename, status)        
+        #self.shutdown = True
diff --git a/module/network/Request.py b/module/network/Request.py
index 3325d085d..07b0129d2 100755
--- a/module/network/Request.py
+++ b/module/network/Request.py
@@ -7,6 +7,8 @@ import urllib
 import urllib2
 import cookielib
 import Keepalive
+import base64
+import time
 
 from Keepalive import HTTPHandler
 from cStringIO import StringIO
@@ -21,23 +23,19 @@ from gzip import GzipFile
     retrieveUrl returns response as string
     
 """
-class Downloader(urllib.FancyURLopener):
-    version = "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8"
-
-
-
 class Request:
     def __init__(self):
+
+        self.dl_time = 0
+        self.dl_size = 0
+        self.dl_arrived = 0
+        self.dl = False
 	
 	self.lastURL = None
-	#self.cookiefile = 'cookies.lwp'
 	self.cj = cookielib.CookieJar()
-
-#	if os.path.isfile(self.cookiefile):
- #           self.cj.load(self.cookiefile)
-
-        self.handler = HTTPHandler()
-	self.opener = urllib2.build_opener(self.handler, urllib2.HTTPCookieProcessor(self.cj))
+        handler = HTTPHandler()
+	self.opener = urllib2.build_opener(handler, urllib2.HTTPCookieProcessor(self.cj))
+	self.downloader = urllib2.build_opener()
 	#self.opener.add_handler()
 	
 	self.opener.addheaders = [
@@ -47,11 +45,15 @@ class Request:
         ("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7"),
 	("Connection","keep-alive"),
         ("Keep-Alive","300")]
-	
-	self.downloader = Downloader()
+
+	self.downloader.addheaders = [
+        ("User-Agent","Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8"),
+        ("Accept-Encoding","gzip,deflate"),
+        ("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"),
+        ("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7")]
 	
     
-    def retrieveURL(self,url, get = {}, post = {}, ref = True):
+    def load(self, url, get = {}, post = {}, ref = True):
 	
 	if post:
 	    post = urllib.urlencode(post)
@@ -72,11 +74,7 @@ class Request:
 	rep = self.opener.open(req)
 	
 	output = rep.read()
-	
-	print rep.headers
-		
-	self.cj.extract_cookies(rep, req)
-	
+				
 	if rep.headers.has_key("content-encoding") :
 	    if rep.headers["content-encoding"] == "gzip" :
 		output = GzipFile('','r',0,StringIO(output)).read()
@@ -86,13 +84,42 @@ class Request:
 	return output
 
     def addAuth(self, user, pw):
-        auth_handler = urllib2.HTTPBasicAuthHandler()
-	auth_handler.add_password(user, passwd= pw)
-	self.opener.add_handler(auth_handler)
+        self.downloader.addheaders.append(['Authorization','Basic ' + base64.encodestring(user + ':' + pw)[:-1]])
+
+
+    #def download(url, filename, reporthook = None, data = None): #default von urlretrieve auch None?
+     #  return self.downloader.urlretrieve(url, filename, reporthook, data)
+
+    def download(self, url, filename):
+        if not self.dl:
+            self.dl = True
+            file = open(filename, 'wb')
+            req = urllib2.Request(url)
+            conn = self.downloader.open(req)
+            self.dl_size = int(conn.headers["content-length"])
+            self.dl_arrived = 0
+            self.dl_time = time.time()
+            for chunk in conn:              
+                self.dl_arrived += len(chunk)    
+                file.write(chunk)
+            file.close()
+            self.dl = False
+            return True
+     
+    def getSpeed(self):
+        try:
+            return (self.dl_arrived / (time.time() - self.dl_time)) / 1024
+        except:
+            return "No Download"
 
-    def download(url, filename, reporthook = None, data = None): #default von urlretrieve auch None?
-       return self.downloader.urlretrieve(url, filename, reporthook, data)
+    def getETA(self):
+        try:
+            return (self.dl_size - self.dl_arrived) / (self.dl_arrived / (time.time() - self.dl_time)) 
+        except:
+            return "No Download"
 
+    def kBleft(self):
+        return (self.dl_size - self.dl_arrived) / 1024
 
 if __name__ == "__main__" :
     import doctest
diff --git a/thread_list.py b/module/thread_list.py
index 0dd61ec7a..2a5dbe6f6 100644
--- a/thread_list.py
+++ b/module/thread_list.py
@@ -55,6 +55,11 @@ class Thread_List(object):
             while not self.status_queue.empty():
                 status = self.status_queue.get()
                 self.py_load_files[status.id].status = status
+
+    def getJob(self):
+        # nur wenn auch geladen werden soll, ansonsten thread in leerlauf schicken
+        if True:
+            return self.download_queue.get()
     
     def extend_py_load_files(self):
         pass