diff options
| -rw-r--r-- | config | 3 | ||||
| -rwxr-xr-x | module/network/Request.py | 4 | ||||
| -rw-r--r-- | module/plugins/FreakshareNet.py | 144 | ||||
| -rw-r--r-- | module/thread_list.py | 2 | ||||
| -rwxr-xr-x | pyLoadCore.py | 92 |
5 files changed, 191 insertions, 54 deletions
@@ -30,7 +30,8 @@ link_file = links.txt failed_file = failed_links.txt reconnect_method = reconnect_method debug_mode = False -max_download_time = 18000 +#hours +max_download_time = 5 [updates] search_updates = True diff --git a/module/network/Request.py b/module/network/Request.py index 9f27af4ec..b80ea44da 100755 --- a/module/network/Request.py +++ b/module/network/Request.py @@ -51,7 +51,7 @@ class Request: self.lastURL = None self.auth = False - self.timeout = 5*3600 + self.timeout = 5 try: if pycurl: self.curl = True @@ -94,7 +94,7 @@ class Request: self.pycurl = pycurl.Curl() self.pycurl.setopt(pycurl.FOLLOWLOCATION, 1) self.pycurl.setopt(pycurl.MAXREDIRS, 5) - self.pycurl.setopt(pycurl.TIMEOUT, self.timeout) + self.pycurl.setopt(pycurl.TIMEOUT, (self.timeout*3600)) self.pycurl.setopt(pycurl.CONNECTTIMEOUT, 30) self.pycurl.setopt(pycurl.NOSIGNAL, 1) self.pycurl.setopt(pycurl.NOPROGRESS, 0) diff --git a/module/plugins/FreakshareNet.py b/module/plugins/FreakshareNet.py new file mode 100644 index 000000000..0768b5476 --- /dev/null +++ b/module/plugins/FreakshareNet.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import re +import urllib +import httplib +from module.Plugin import Plugin +from time import time + + +class FreakshareNet(Plugin): + + def __init__(self, parent): + Plugin.__init__(self, parent) + props = {} + props['name'] = "FreakshareNet" + props['type'] = "hoster" + props['pattern'] = r"http://(?:www\.)?freakshare\.net/files/\S*?/" + props['version'] = "0.1" + props['description'] = """Freakshare.com Download Plugin""" + props['author_name'] = ("sitacuisses","spoob","mkaay") + props['author_mail'] = ("sitacuisses@yahoo.de","spoob@pyload.org","mkaay@mkaay.de") + self.props = props + self.parent = parent + self.html = None + self.want_reconnect = False + self.multi_dl = False + self.req_opts = list() + + def prepare(self, thread): + pyfile = self.parent + + self.want_reconnect = False + + self.download_html() + + pyfile.status.exists = self.file_exists() + + if not pyfile.status.exists: + raise Exception, "The file was not found on the server." + return False + + self.get_waiting_time() + + pyfile.status.filename = self.get_file_name() + + pyfile.status.waituntil = self.time_plus_wait + thread.wait(self.parent) + pyfile.status.url = self.get_file_url() + pyfile.status.want_reconnect = self.want_reconnect + + return True + + def download_html(self): + url = self.parent.url + self.html = self.req.load(url, cookies=True) + + def get_file_url(self): + """ returns the absolute downloadable filepath + """ + if self.html == None: + self.download_html() + if not self.want_reconnect: + self.req_opts = self.get_download_options() # get the Post options for the Request + file_url = self.parent.url + return file_url + else: + return False + + def get_file_name(self): + if self.html == None: + self.download_html() + if not self.want_reconnect: + file_name = re.search(r"<h1\sclass=\"box_heading\"\sstyle=\"text-align:center\;\">(.*?)<\/h1>", self.html).group(1) + return file_name + else: + return self.parent.url + + def get_waiting_time(self): + if self.html == None: + self.download_html() + timestring = re.search('\s*var\stime\s=\s(\d*?)\.\d*;', self.html).group(1) + if timestring: + sec = int(timestring) + 1 #add 1 sec as tenths of seconds are cut off + else: + sec = 0 + self.time_plus_wait = time() + sec + + def file_exists(self): + """ returns True or False + """ + if self.html == None: + self.download_html() + if re.search(r"Sorry, this Download doesnt exist anymore", self.html) != None: + return False + else: + return True + + def get_download_options(self): + re_envelope = re.search(r".*?value=\"Free\sDownload\".*?\n*?(.*?<.*?>\n*)*?\n*\s*?</form>", self.html).group(0) #get the whole request + to_sort = re.findall(r"<input\stype=\"hidden\"\svalue=\"(.*?)\"\sname=\"(.*?)\"\s\/>", re_envelope) + request_options = list() + for item in to_sort: #Name value pairs are output reversed from regex, so we reorder them + request_options.append((item[1], item[0])) + herewego = self.req.load(self.parent.url, None, request_options, cookies=True) # the actual download-Page + to_sort = None + to_sort = re.findall(r"<input\stype=\".*?\"\svalue=\"(\S*?)\".*?name=\"(\S*?)\"\s.*?\/>", herewego) + request_options = list() + for item in to_sort: #Same as above + request_options.append((item[1], item[0])) + return request_options + + def proceed(self, url, location): + """ + request.download doesn't handle the 302 redirect correctly + that's why the data are posted "manually" via httplib + and the redirect-url is read from the header. + Important: The cookies may not be posted to the download-url + otherwise the downloaded file only contains "bad try" + Need to come up with a better idea to handle the redirect, + help is appreciated. + """ + temp_options = urllib.urlencode(self.req_opts) + temp_url = re.match(r"http://(.*?)/.*", url).group(1) # get the server name + temp_extended = re.match(r"http://.*?(/.*)", url).group(1) # get the url relative to serverroot + cookie_list = "" + for temp_cookie in self.req.cookies: #prepare cookies + cookie_list += temp_cookie.name + "=" + temp_cookie.value +";" + temp_headers = [ #create the additional header fields + ["Content-type", "application/x-www-form-urlencoded"], #this is very important + ["User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.10"], + ["Accept-Encoding", "deflate"], + ["Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"], + ["Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7"], + ["Connection", "keep-alive"], + ["Keep-Alive", "300"], + ["Referer", self.req.lastURL], + ["Cookie", cookie_list]] + temp_conn = httplib.HTTPConnection(temp_url) + temp_conn.request("POST", temp_extended, temp_options, dict(temp_headers)) + temp_response = temp_conn.getresponse() + new_url = temp_response.getheader("Location") # we need the Location-header + temp_conn.close + self.req.download(new_url, location, None, None, cookies=False) diff --git a/module/thread_list.py b/module/thread_list.py index aa50a654f..fc886e4b4 100644 --- a/module/thread_list.py +++ b/module/thread_list.py @@ -60,7 +60,7 @@ class Thread_List(object): def get_job(self): """return job if suitable, otherwise send thread idle""" - if not self.parent.is_time_download() or self.pause or self.reconnecting or self.list.queueEmpty(): #conditions when threads dont download + if not self.parent.server_methods.is_time_download() or self.pause or self.reconnecting or self.list.queueEmpty(): #conditions when threads dont download return None self.init_reconnect() diff --git a/pyLoadCore.py b/pyLoadCore.py index be9232300..add30126b 100755 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -134,17 +134,18 @@ class Core(object): self.check_update() - self.logger.info(_("Downloadtime: %s") % self.is_time_download()) # debug only - path.append(self.plugin_folder) self.create_plugin_index() - self.init_server() - + self.server_methods = ServerMethods(self) self.file_list = File_List(self) self.thread_list = Thread_List(self) - self.read_url_list(self.config['general']['link_file']) + self.init_server() + + self.logger.info(_("Downloadtime: %s") % self.server_methods.is_time_download()) # debug only + + self.server_methods.read_url_list(self.config['general']['link_file']) while True: sleep(2) @@ -165,21 +166,8 @@ class Core(object): self.server = Server.SimpleXMLRPCServer(server_addr) self.logger.info("Normal XMLRPC Server Started") - self.server.register_introspection_functions() - self.server.register_function(self.status_downloads) - self.server.register_function(self.status_server) - self.server.register_function(self.kill) - self.server.register_function(self.del_links) - self.server.register_function(self.del_packages) - self.server.register_function(self.add_urls) - self.server.register_function(self.get_queue) - #self.server.register_function(self.move_urls_up) - #self.server.register_function(self.move_urls_down) - self.server.register_function(self.is_time_download) - self.server.register_function(self.is_time_reconnect) - self.server.register_function(self.get_conf_val) - self.server.register_function(self.file_exists) - self.server.register_function(self.get_server_version) + self.server.register_instance(self.server_methods) + thread.start_new_thread(self.server.serve_forever, ()) except Exception, e: self.logger.error("Failed starting socket server, CLI and GUI will not be available: %s" % str(e)) @@ -275,10 +263,14 @@ class Core(object): #################################### ########## XMLRPC Methods ########## #################################### - + +class ServerMethods(): + def __init__(self, core): + self.core = core + def status_downloads(self): downloads = [] - for pyfile in self.thread_list.py_downloading: + for pyfile in self.core.thread_list.py_downloading: download = {} download['id'] = pyfile.id download['name'] = pyfile.status.filename @@ -295,17 +287,17 @@ class Core(object): def get_conf_val(self, cat, var): if var != "username" and var != "password": - return self.config[cat][var] + return self.core.config[cat][var] else: raise Exception("not allowed!") def status_server(self): status = {} - status['pause'] = self.thread_list.pause - status['queue'] = len(self.file_list.files) + status['pause'] = self.core.thread_list.pause + status['queue'] = len(self.core.file_list.files) status['speed'] = 0 - for pyfile in self.thread_list.py_downloading: + for pyfile in self.core.thread_list.py_downloading: status['speed'] += pyfile.status.get_speed() return status @@ -318,29 +310,29 @@ class Core(object): def add_urls(self, links): for link in links: - self.file_list.collector.addLink(link) - self.file_list.save() + self.core.file_list.collector.addLink(link) + self.core.file_list.save() def del_links(self, ids): for id in ids: try: - self.file_list.collector.removeFile(id) + self.core.file_list.collector.removeFile(id) except: - self.file_list.packages.removeFile(id) - self.file_list.save() + self.core.file_list.packages.removeFile(id) + self.core.file_list.save() def del_packages(self, ids): for id in ids: - self.file_list.packages.removePackage(id) - self.file_list.save() + self.core.file_list.packages.removePackage(id) + self.core.file_list.save() def kill(self): - self.do_kill = True + self.core.do_kill = True return True def get_queue(self): data = [] - for q in self.file_list.data["queue"]: + for q in self.core.file_list.data["queue"]: ds = { "id": q.data.id, "name": q.data.package_name, @@ -358,7 +350,7 @@ class Core(object): def get_collector_packages(self): data = [] - for q in self.file_list.data["packages"]: + for q in self.core.file_list.data["packages"]: ds = { "id": q.data.id, "name": q.data.package_name, @@ -376,42 +368,42 @@ class Core(object): #def move_urls_up(self, ids): # for id in ids: - # self.file_list.move(id) - # self.file_list.save() + # self.core.file_list.move(id) + # self.core.file_list.save() #def move_urls_down(self, ids): # for id in ids: - # self.file_list.move(id, 1) - # self.file_list.save() + # self.core.file_list.move(id, 1) + # self.core.file_list.save() def read_url_list(self, url_list): """read links from txt""" - txt = open(self.config['general']['link_file'], 'r') + txt = open(self.core.config['general']['link_file'], 'r') new_links = 0 links = txt.readlines() for link in links: if link != "\n": - self.file_list.collector.addLink(link) + self.core.file_list.collector.addLink(link) new_links += 1 txt.close() - self.file_list.save() + self.core.file_list.save() if new_links: - self.logger.info("Parsed link from %s: %i" % (self.config['general']['link_file'], new_links)) + self.core.logger.info("Parsed link from %s: %i" % (self.core.config['general']['link_file'], new_links)) - txt = open(self.config['general']['link_file'], 'w') + txt = open(self.core.config['general']['link_file'], 'w') txt.write("") txt.close() def is_time_download(self): - start = self.config['downloadTime']['start'].split(":") - end = self.config['downloadTime']['end'].split(":") - return self.compare_time(start, end) + start = self.core.config['downloadTime']['start'].split(":") + end = self.core.config['downloadTime']['end'].split(":") + return self.core.compare_time(start, end) def is_time_reconnect(self): - start = self.config['reconnectTime']['start'].split(":") - end = self.config['reconnectTime']['end'].split(":") + start = self.core.config['reconnectTime']['start'].split(":") + end = self.core.config['reconnectTime']['end'].split(":") return self.compare_time(start, end) if __name__ == "__main__": |