diff options
author | RaNaN <Mast3rRaNaN@hotmail.de> | 2009-08-10 10:43:57 +0200 |
---|---|---|
committer | RaNaN <Mast3rRaNaN@hotmail.de> | 2009-08-10 10:43:57 +0200 |
commit | 2e76cf0f532ff89ffa58e5db264a14deb60db6da (patch) | |
tree | 7732fdbdceb957e3596ee275815957ecc2d84045 | |
parent | fixed megaupload bug (diff) | |
download | pyload-2e76cf0f532ff89ffa58e5db264a14deb60db6da.tar.xz |
plugin fixes, core config functions
-rw-r--r-- | Plugins/NetloadIn.py | 50 | ||||
-rw-r--r-- | Plugins/RapidshareCom.py | 14 | ||||
-rw-r--r-- | Plugins/UploadedTo.py | 46 | ||||
-rw-r--r-- | captcha/captcha.py | 313 | ||||
-rw-r--r-- | config | 39 | ||||
-rw-r--r-- | pyLoadCore.py | 84 |
6 files changed, 281 insertions, 265 deletions
diff --git a/Plugins/NetloadIn.py b/Plugins/NetloadIn.py index 0df1fab8c..ac14a670c 100644 --- a/Plugins/NetloadIn.py +++ b/Plugins/NetloadIn.py @@ -33,26 +33,35 @@ class NetloadIn(Plugin): self.want_reconnect = False - self.req.clear_cookies() - self.download_html() + tries = 0 - pyfile.status.exists = self.file_exists() + while not pyfile.status.url: - if not pyfile.status.exists: - raise Exception, "The file was not found on the server." + self.req.clear_cookies() + self.download_html() + + pyfile.status.exists = self.file_exists() + + if not pyfile.status.exists: + raise Exception, "The file was not found on the server." - pyfile.status.filename = self.get_file_name() + pyfile.status.filename = self.get_file_name() - self.download_html2() + self.download_html2() - self.get_wait_time() + self.get_wait_time() - pyfile.status.waituntil = self.time_plus_wait - pyfile.status.want_reconnect = self.want_reconnect + pyfile.status.waituntil = self.time_plus_wait + pyfile.status.want_reconnect = self.want_reconnect - thread.wait(self.parent) + thread.wait(self.parent) + + pyfile.status.url = self.get_file_url() + + tries += 1 + if tries > 3: + raise Exception, "Error while preparing DL, HTML dump: %s %s" % (self.html[0], self.html[1]) - pyfile.status.url = self.get_file_url() return True @@ -72,24 +81,25 @@ class NetloadIn(Plugin): for i in range(10): self.req.download(captcha_url, captcha_image, cookies=True) captcha = self.ocr.get_captcha(captcha_image) + self.logger.debug("Captcha %s: %s" % (i, captcha)) sleep(5) self.html[2] = self.req.load("http://netload.in/index.php?id=10", post={"file_id": file_id, "captcha_check": captcha}, cookies=True) if re.search(r"(We will prepare your download..|We had a reqeust with the IP)", self.html[2]) != None: - break + return True + + raise Exception, "Captcha reading failed" os.remove(captcha_image) def get_file_url(self): """ returns the absolute downloadable filepath """ - if self.html[0] == None: - self.download_html() - if not self.want_reconnect: + try: file_url_pattern = r"<a class=\"Orange_Link\" href=\"(http://.+)\" >Click here" search = re.search(file_url_pattern, self.html[2]) return search.group(1) - else: - return False + except: + return None def get_wait_time(self): wait = int(re.search(r"countdown\((.+),'change\(\)'\)", self.html[2]).group(1)) @@ -99,8 +109,6 @@ class NetloadIn(Plugin): self.want_reconnect = True def get_file_name(self): - if self.html[0] == None: - self.download_html() if not self.want_reconnect: file_name_pattern = '\t\t\t(.+)<span style="color: #8d8d8d;">' return re.search(file_name_pattern, self.html[0]).group(1) @@ -110,8 +118,6 @@ class NetloadIn(Plugin): def file_exists(self): """ returns True or False """ - if self.html[0] == None: - self.download_html() if re.search(r"The file has been deleted", self.html[0]) != None: return False else: diff --git a/Plugins/RapidshareCom.py b/Plugins/RapidshareCom.py index 52d731053..2c78a2672 100644 --- a/Plugins/RapidshareCom.py +++ b/Plugins/RapidshareCom.py @@ -62,7 +62,7 @@ class RapidshareCom(Plugin): tries += 1 if tries > 5: - raise Exception, "Error when downloading, HTML dump:"+ str(self.html[0]) + str(self.html[1]) + raise Exception, "Error while preparing, HTML dump:"+ str(self.html[0]) + str(self.html[1]) return True @@ -78,8 +78,6 @@ class RapidshareCom(Plugin): def download_serverhtml(self): """downloads html with the important informations """ - if self.html[0] == None: - self.download_html() if self.config['premium']: return False @@ -95,9 +93,6 @@ class RapidshareCom(Plugin): self.time_plus_wait = 0 return True - if self.html[1] == None: - self.download_serverhtml(self) - if re.search(r".*is already downloading.*", self.html[1]) != None: self.time_plus_wait = time() + 10 * 60 try: @@ -114,8 +109,6 @@ class RapidshareCom(Plugin): def file_exists(self): """ returns True or False """ - if self.html[0] == None: - self.download_html() if re.search(r".*The File could not be found.*", self.html[0]) != None or \ re.search(r"(<p>This limit is reached.</p>)", self.html[0]) or \ re.search(r"(.*is momentarily not available.*)", self.html[0]) or \ @@ -132,9 +125,6 @@ class RapidshareCom(Plugin): self.start_dl = True return self.parent.url - if self.html[1] == None: - self.download_serverhtml() - #if (self.html_old + 5 * 60) < time(): # nach einiger zeit ist die file_url nicht mehr aktuell # self.download_serverhtml() @@ -154,8 +144,6 @@ class RapidshareCom(Plugin): #raise Exception, "Error when retrieving download url" def get_file_name(self): - if self.html[0] == None: - self.download_html() file_name_pattern = r"<p class=\"downloadlink\">.+/(.+) <font" return re.findall(file_name_pattern, self.html[0])[0] diff --git a/Plugins/UploadedTo.py b/Plugins/UploadedTo.py index 7f30a0519..e163a3230 100644 --- a/Plugins/UploadedTo.py +++ b/Plugins/UploadedTo.py @@ -25,6 +25,35 @@ class UploadedTo(Plugin): self.want_reconnect = None self.multi_dl = False + def prepare(self, thread): + pyfile = self.parent + + self.want_reconnect = False + tries = 0 + + while not pyfile.status.url: + + self.download_html() + + pyfile.status.exists = self.file_exists() + + if not pyfile.status.exists: + raise Exception, "The file was not found on the server." + + pyfile.status.filename = self.get_file_name() + + pyfile.status.waituntil = self.time_plus_wait + pyfile.status.url = self.get_file_url() + pyfile.status.want_reconnect = self.want_reconnect + + thread.wait(self.parent) + + tries += 1 + if tries > 5: + raise Exception, "Error while preparing DL, HTML dump: %s" % self.html + + return True + def download_html(self): url = self.parent.url self.html = self.req.load(url) @@ -39,17 +68,13 @@ class UploadedTo(Plugin): def get_file_url(self): """ returns the absolute downloadable filepath """ - if self.html == None: - self.download_html() - if not self.want_reconnect: + try: file_url_pattern = r".*<form name=\"download_form\" method=\"post\" action=\"(.*)\">" return re.search(file_url_pattern, self.html).group(1) - else: - return False + except: + return None def get_file_name(self): - if self.html == None: - self.download_html() if not self.want_reconnect: file_name = re.search(r"<td><b>\s+(.+)\s", self.html).group(1) file_suffix = re.search(r"</td><td>(\..+)</td></tr>", self.html) @@ -62,14 +87,7 @@ class UploadedTo(Plugin): def file_exists(self): """ returns True or False """ - if self.html == None: - self.download_html() if re.search(r"(File doesn't exist .*)", self.html) != None: return False else: return True - - def wait_until(self): - if self.html == None: - self.download_html() - return self.time_plus_wait diff --git a/captcha/captcha.py b/captcha/captcha.py index 5fa8bfc45..fbbf20fee 100644 --- a/captcha/captcha.py +++ b/captcha/captcha.py @@ -17,11 +17,12 @@ # along with this program; if not, see <http://www.gnu.org/licenses/>. # ### +import logging import subprocess import tempfile +import threading import Image -import threading class RunThread(threading.Thread): def __init__(self): @@ -47,7 +48,7 @@ class RunThread(threading.Thread): class OCR(object): def __init__(self): - pass + self.logger = logging.getLogger("log") def load_image(self, image): self.image = Image.open(image) @@ -71,207 +72,207 @@ class OCR(object): # "Error running: %s\n\n%s" % (command, errdata) # return outputdata - thread = RunThread() - result = thread.e(command, inputdata) - return result + thread = RunThread() + result = thread.e(command, inputdata) + return result - def run_gocr(self): - tmp = tempfile.NamedTemporaryFile(suffix=".jpg") - self.image.save(tmp) - self.result_captcha = self.run(['gocr', tmp.name]).replace("\n", "") +def run_gocr(self): + tmp = tempfile.NamedTemporaryFile(suffix=".jpg") + self.image.save(tmp) + self.result_captcha = self.run(['gocr', tmp.name]).replace("\n", "") - def run_tesser(self): - tmp = tempfile.NamedTemporaryFile(suffix=".tif") - tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") +def run_tesser(self): + tmp = tempfile.NamedTemporaryFile(suffix=".tif") + tmpTxt = tempfile.NamedTemporaryFile(suffix=".txt") - self.image.save(tmp.name, 'TIFF') - self.run(['tesseract', tmp.name, tmpTxt.name.replace(".txt", "")]) + self.image.save(tmp.name, 'TIFF') + self.run(['tesseract', tmp.name, tmpTxt.name.replace(".txt", "")]) - self.result_captcha = self.run(['cat', tmpTxt.name]).replace("\n", "") + self.result_captcha = self.run(['cat', tmpTxt.name]).replace("\n", "") - def get_captcha(self): - raise NotImplementedError +def get_captcha(self): + raise NotImplementedError - def to_greyscale(self): - if self.image.mode != 'L': - self.image = self.image.convert('L') +def to_greyscale(self): + if self.image.mode != 'L': + self.image = self.image.convert('L') - self.pixels = self.image.load() + self.pixels = self.image.load() - def eval_black_white(self, limit): - self.pixels = self.image.load() - w, h = self.image.size - for x in xrange(w): - for y in xrange(h): - if self.pixels[x, y] > limit: - self.pixels[x, y] = 255 - else: - self.pixels[x, y] = 0 +def eval_black_white(self, limit): + self.pixels = self.image.load() + w, h = self.image.size + for x in xrange(w): + for y in xrange(h): + if self.pixels[x, y] > limit: + self.pixels[x, y] = 255 + else: + self.pixels[x, y] = 0 - def clean(self, allowed): - pixels = self.pixels +def clean(self, allowed): + pixels = self.pixels - w, h = self.image.size + w, h = self.image.size - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 255: continue - # no point in processing white pixels since we only want to remove black pixel - count = 0 - - try: - if pixels[x-1, y-1] != 255: count += 1 - if pixels[x-1, y] != 255: count += 1 - if pixels[x-1, y + 1] != 255: count += 1 - if pixels[x, y + 1] != 255: count += 1 - if pixels[x + 1, y + 1] != 255: count += 1 - if pixels[x + 1, y] != 255: count += 1 - if pixels[x + 1, y-1] != 255: count += 1 - if pixels[x, y-1] != 255: count += 1 - except: - pass - - # not enough neighbors are dark pixels so mark this pixel - # to be changed to white - if count < allowed: - pixels[x, y] = 1 - - # second pass: this time set all 1's to 255 (white) - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 1: pixels[x, y] = 255 + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 255: continue + # no point in processing white pixels since we only want to remove black pixel + count = 0 - self.pixels = pixels + try: + if pixels[x-1, y-1] != 255: count += 1 + if pixels[x-1, y] != 255: count += 1 + if pixels[x-1, y + 1] != 255: count += 1 + if pixels[x, y + 1] != 255: count += 1 + if pixels[x + 1, y + 1] != 255: count += 1 + if pixels[x + 1, y] != 255: count += 1 + if pixels[x + 1, y-1] != 255: count += 1 + if pixels[x, y-1] != 255: count += 1 + except: + pass - def derotate_by_average(self): - """rotate by checking each angle and guess most suitable""" + # not enough neighbors are dark pixels so mark this pixel + # to be changed to white + if count < allowed: + pixels[x, y] = 1 - w, h = self.image.size - pixels = self.pixels + # second pass: this time set all 1's to 255 (white) + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 1: pixels[x, y] = 255 - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 0: - pixels[x, y] = 155 + self.pixels = pixels + +def derotate_by_average(self): + """rotate by checking each angle and guess most suitable""" - highest = {} - counts = {} + w, h = self.image.size + pixels = self.pixels - for angle in range(-45, 45): + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 155 - tmpimage = self.image.rotate(angle) + highest = {} + counts = {} + + for angle in range(-45, 45): + + tmpimage = self.image.rotate(angle) - pixels = tmpimage.load() + pixels = tmpimage.load() - w, h = self.image.size + w, h = self.image.size - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 0: - pixels[x, y] = 255 + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 - count = {} + count = {} - for x in xrange(w): - count[x] = 0 - for y in xrange(h): - if pixels[x, y] == 155: - count[x] += 1 + for x in xrange(w): + count[x] = 0 + for y in xrange(h): + if pixels[x, y] == 155: + count[x] += 1 - sum = 0 - cnt = 0 + sum = 0 + cnt = 0 - for x in count.values(): - if x != 0: - sum += x - cnt += 1 + for x in count.values(): + if x != 0: + sum += x + cnt += 1 - avg = sum / cnt - counts[angle] = cnt - highest[angle] = 0 - for x in count.values(): - if x > highest[angle]: - highest[angle] = x + avg = sum / cnt + counts[angle] = cnt + highest[angle] = 0 + for x in count.values(): + if x > highest[angle]: + highest[angle] = x - highest[angle] = highest[angle] - avg + highest[angle] = highest[angle] - avg - hkey = 0 - hvalue = 0 + hkey = 0 + hvalue = 0 - for key, value in highest.iteritems(): - if value > hvalue: - hkey = key - hvalue = value + for key, value in highest.iteritems(): + if value > hvalue: + hkey = key + hvalue = value - self.image = self.image.rotate(hkey) - pixels = self.image.load() + self.image = self.image.rotate(hkey) + pixels = self.image.load() - for x in xrange(w): - for y in xrange(h): - if pixels[x, y] == 0: - pixels[x, y] = 255 + for x in xrange(w): + for y in xrange(h): + if pixels[x, y] == 0: + pixels[x, y] = 255 - if pixels[x, y] == 155: - pixels[x, y] = 0 + if pixels[x, y] == 155: + pixels[x, y] = 0 - self.pixels = pixels + self.pixels = pixels - def split_captcha_letters(self): - captcha = self.image - started = False - letters = [] - width, height = captcha.size - bottomY, topY = 0, height - pixels = captcha.load() +def split_captcha_letters(self): + captcha = self.image + started = False + letters = [] + width, height = captcha.size + bottomY, topY = 0, height + pixels = captcha.load() - for x in xrange(width): - black_pixel_in_col = False - for y in xrange(height): - if pixels[x, y] != 255: - if started == False: - started = True - firstX = x - lastX = x + for x in xrange(width): + black_pixel_in_col = False + for y in xrange(height): + if pixels[x, y] != 255: + if started == False: + started = True + firstX = x + lastX = x - if y > bottomY: bottomY = y - if y < topY: topY = y - if x > lastX: lastX = x + if y > bottomY: bottomY = y + if y < topY: topY = y + if x > lastX: lastX = x - black_pixel_in_col = True + black_pixel_in_col = True - if black_pixel_in_col == False and started == True: - rect = (firstX, topY, lastX, bottomY) - new_captcha = captcha.crop(rect) + if black_pixel_in_col == False and started == True: + rect = (firstX, topY, lastX, bottomY) + new_captcha = captcha.crop(rect) - w, h = new_captcha.size - if w > 5 and h > 5: - letters.append(new_captcha) + w, h = new_captcha.size + if w > 5 and h > 5: + letters.append(new_captcha) - started = False - bottomY, topY = 0, height + started = False + bottomY, topY = 0, height - return letters + return letters - def correct(self, values, var=None): +def correct(self, values, var=None): - if var: - result = var - else: - result = self.result_captcha - - for key, item in values.iteritems(): + if var: + result = var + else: + result = self.result_captcha - if key.__class__ == str: - result = result.replace(key, item) - else: - for expr in key: - result = result.replace(expr, item) + for key, item in values.iteritems(): - if var: - return result + if key.__class__ == str: + result = result.replace(key, item) else: - self.result_captcha = result + for expr in key: + result = result.replace(expr, item) + + if var: + return result + else: + self.result_captcha = result if __name__ == '__main__': @@ -1,37 +1,34 @@ +[remote] +port = 7272 +remotepassword = pwhere + +[log] +file_log = True +log_folder = Logs +log_count = 5 + [general] -#use en or de -language: de +language = de download_folder = Downloads +max_downloads = 3 +use_reconnect = False link_file = links.txt failed_file = failed_links.txt -use_reconnect = False -reconnect_method = reconnect_method -max_downloads = 3 +reconnect_method = reconnect_method [updates] search_updates = True install_updates = False -[log] -file_log = True -log_folder = Logs -log_count = 5 -#each log has 100kb +[reconnectTime] +starttime = 0:00 +endtime = 0:00 [downloadTime] -#use HH:MM start = 0:00 end = 0:00 -[reconnectTime] -startTime = 0:00 -endTime = 0:00 - -[remote] -port = 7272 -remotePassword = pwhere - [proxy] useproxy = False -proxyProtocol = http -proxyAdress = http://localhost:8080 +proxyadress = http://localhost:8080 +proxyprotocol = http
\ No newline at end of file diff --git a/pyLoadCore.py b/pyLoadCore.py index 079a40ec7..8587aea5d 100644 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -80,14 +80,23 @@ class Core(object): def read_config(self): """ read config and sets preferences """ - config = ConfigParser.SafeConfigParser() - config.read('config') + self.configfile = ConfigParser.SafeConfigParser() + self.configfile.read('config') - for section in config.sections(): - for option in config.options(section): - self.config[option] = config.get(section, option) + for section in self.configfile.sections(): + for option in self.configfile.options(section): + self.config[option] = self.configfile.get(section, option) self.config[option] = False if self.config[option].lower() == 'false' else self.config[option] + def set_option(self, section, option, value): + self.config[option] = value + self.configfile.set(section, option, str(value)) + self.configfile.write(open('config', "wb")) + + def read_option(self): + return self.config + + def create_plugin_index(self): for file_handler in glob(self.config['plugin_folder'] + sep + '*.py') + glob(self.config['plugin_folder'] + sep + 'DLC.pyc'): if file_handler != self.config['plugin_folder'] + sep + "Plugin.py": @@ -167,36 +176,40 @@ class Core(object): self.logger.setLevel(level) def is_dltime(self): - start_h, start_m = self.config['start'].split(":") - end_h, end_m = self.config['end'].split(":") + start = self.config['start'].split(":") + end = self.config['end'].split(":") - return self.compare_time(start_h, start_m, end_h, end_m) + return self.compare_time(start, end) def is_reconnect_time(self): - start_h, start_m = self.config['starttime'].split(":") - end_h, end_m = self.config['endtime'].split(":") + start = self.config['starttime'].split(":") + end = self.config['endtime'].split(":") - return self.compare_time(start_h, start_m, end_h, end_m) + return self.compare_time(start, end) - def compare_time(self, start_h, start_m, end_h, end_m): + def compare_time(self, start, end): - if (start_h, start_m) == (end_h, end_m): + if start == end: return True - hour, minute = time.localtime()[3:5] + now = time.localtime()[3:5] - if hour > int(start_h) and hour < int(end_h): + if start < now and end > now: return True - elif hour < int(end_h) and int(start_h) > int(end_h): - return True - elif hour == int(start_h) and minute >= int(start_m): - return True - elif hour == int(end_h) and minute <= int(end_m): + elif start < now and end < now and start > end: return True else: return False + def format_time(self, seconds): + seconds = int(seconds) + if seconds > 60: + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) + return _("%i seconds") % seconds + def get_downloads(self): list = [] for pyfile in self.thread_list.py_downloading: @@ -214,25 +227,6 @@ class Core(object): return list - def format_time(self, seconds): - seconds = int(seconds) - if seconds > 60: - hours, seconds = divmod(seconds, 3600) - minutes, seconds = divmod(seconds, 60) - return "%.2i:%.2i:%.2i" % (hours, minutes, seconds) - return _("%i seconds") % seconds - - def _test_print_status(self): - - if self.thread_list.py_downloading: - - for pyfile in self.thread_list.py_downloading: - if pyfile.status.type == 'downloading': - print pyfile.status.filename + ": speed is", int(pyfile.status.get_speed()), "kb/s" - print pyfile.status.filename + ": finished in", self.format_time(pyfile.status.get_ETA()) - elif pyfile.status.type == 'waiting': - print pyfile.status.filename + ": wait", self.format_time(pyfile.status.waituntil - time.time()) - def server_send_status(self): obj = RequestObject() obj.command = "update" @@ -306,6 +300,7 @@ class Core(object): """ starts the machine """ self.read_links() + while True: #self.thread_list.status() if print_test_status: @@ -316,6 +311,17 @@ class Core(object): self.logger.info("pyLoad quits") exit() + def _test_print_status(self): + + if self.thread_list.py_downloading: + + for pyfile in self.thread_list.py_downloading: + if pyfile.status.type == 'downloading': + print pyfile.status.filename + ": speed is", int(pyfile.status.get_speed()), "kb/s" + print pyfile.status.filename + ": finished in", self.format_time(pyfile.status.get_ETA()) + elif pyfile.status.type == 'waiting': + print pyfile.status.filename + ": wait", self.format_time(pyfile.status.waituntil - time.time()) + if __name__ == "__main__": if len(argv) > 1: if argv[1] == "-v": |