From d35c003cc53d4723d1dfe0d81eeb9bea78cee594 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 31 Dec 2011 16:01:24 +0100 Subject: new crypter plugin API, now decrypting possible for now. --- module/threads/DownloadThread.py | 215 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 module/threads/DownloadThread.py (limited to 'module/threads/DownloadThread.py') diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py new file mode 100644 index 000000000..3d444686b --- /dev/null +++ b/module/threads/DownloadThread.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see . + + @author: RaNaN +""" + +from Queue import Queue +from time import sleep, time +from traceback import print_exc +from sys import exc_clear +from pycurl import error + +from module.plugins.Base import Fail, Retry +from module.plugins.Hoster import Abort, Reconnect, SkipDownload + +from BaseThread import BaseThread + +class DownloadThread(BaseThread): + """thread for downloading files from 'real' hoster plugins""" + + def __init__(self, manager): + """Constructor""" + BaseThread.__init__(self, manager) + + self.queue = Queue() # job queue + self.active = False + + self.start() + + def run(self): + """run method""" + pyfile = None + + while True: + del pyfile + self.active = self.queue.get() + pyfile = self.active + + if self.active == "quit": + self.active = False + self.m.threads.remove(self) + return True + + try: + if not pyfile.hasPlugin(): continue + #this pyfile was deleted while queueing + + pyfile.plugin.checkForSameFiles(starting=True) + self.m.log.info(_("Download starts: %s" % pyfile.name)) + + # start download + self.m.core.hookManager.downloadPreparing(pyfile) + pyfile.plugin.preprocessing(self) + + self.m.log.info(_("Download finished: %s") % pyfile.name) + self.m.core.hookManager.downloadFinished(pyfile) + self.m.core.files.checkPackageFinished(pyfile) + + except NotImplementedError: + self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + pyfile.setStatus("failed") + pyfile.error = "Plugin does not work" + self.clean(pyfile) + continue + + except Abort: + try: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + except: + pass + + pyfile.setStatus("aborted") + + self.clean(pyfile) + continue + + except Reconnect: + self.queue.put(pyfile) + #pyfile.req.clearCookies() + + while self.m.reconnecting.isSet(): + sleep(0.5) + + continue + + except Retry, e: + reason = e.args[0] + self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) + self.queue.put(pyfile) + continue + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + pyfile.setStatus("offline") + self.m.log.warning(_("Download is offline: %s") % pyfile.name) + elif msg == "temp. offline": + pyfile.setStatus("temp. offline") + self.m.log.warning(_("Download is temporary offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + + self.m.core.hookManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + except error, e: + if len(e.args) == 2: + code, msg = e.args + else: + code = 0 + msg = e.args + + self.m.log.debug("pycurl exception %s: %s" % (code, msg)) + + if code in (7, 18, 28, 52, 56): + self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) + wait = time() + 60 + + pyfile.waitUntil = wait + pyfile.setStatus("waiting") + while time() < wait: + sleep(1) + if pyfile.abort: + break + + if pyfile.abort: + self.m.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + self.clean(pyfile) + else: + self.queue.put(pyfile) + + continue + + else: + pyfile.setStatus("failed") + self.m.log.error("pycurl error %s: %s" % (code, msg)) + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + self.m.core.hookManager.downloadFailed(pyfile) + + self.clean(pyfile) + continue + + except SkipDownload, e: + pyfile.setStatus("skipped") + + self.m.log.info( + _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) + + self.clean(pyfile) + + self.m.core.files.checkPackageFinished(pyfile) + + self.active = False + self.m.core.files.save() + + continue + + + except Exception, e: + pyfile.setStatus("failed") + self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) + + if self.m.core.debug: + print_exc() + self.writeDebugReport(pyfile) + + self.m.core.hookManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + finally: + self.m.core.files.save() + pyfile.checkIfProcessed() + exc_clear() + + + #pyfile.plugin.req.clean() + + self.active = False + pyfile.finishIfDone() + self.m.core.files.save() + + + def put(self, job): + """assing job to thread""" + self.queue.put(job) + + + def stop(self): + """stops the thread""" + self.put("quit") \ No newline at end of file -- cgit v1.2.3 From 35742c2cb023ac49ab3056752d2040cdb030cc2b Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 1 Jan 2012 13:36:59 +0100 Subject: Happy new Year ! --- module/threads/DownloadThread.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/threads/DownloadThread.py') diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py index 3d444686b..638861338 100644 --- a/module/threads/DownloadThread.py +++ b/module/threads/DownloadThread.py @@ -156,7 +156,7 @@ class DownloadThread(BaseThread): self.m.log.error("pycurl error %s: %s" % (code, msg)) if self.m.core.debug: print_exc() - self.writeDebugReport(pyfile) + self.writeDebugReport(pyfile.pluginname, pyfile) self.m.core.hookManager.downloadFailed(pyfile) @@ -186,7 +186,7 @@ class DownloadThread(BaseThread): if self.m.core.debug: print_exc() - self.writeDebugReport(pyfile) + self.writeDebugReport(pyfile.pluginname, pyfile) self.m.core.hookManager.downloadFailed(pyfile) self.clean(pyfile) -- cgit v1.2.3 From 6eaa7bb25e2254c80c43fe46166142d590e86c64 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sat, 7 Jan 2012 23:58:28 +0100 Subject: some cleanups --- module/threads/DownloadThread.py | 54 ++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 27 deletions(-) (limited to 'module/threads/DownloadThread.py') diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py index 638861338..e140703d5 100644 --- a/module/threads/DownloadThread.py +++ b/module/threads/DownloadThread.py @@ -60,18 +60,18 @@ class DownloadThread(BaseThread): #this pyfile was deleted while queueing pyfile.plugin.checkForSameFiles(starting=True) - self.m.log.info(_("Download starts: %s" % pyfile.name)) + self.log.info(_("Download starts: %s" % pyfile.name)) # start download - self.m.core.hookManager.downloadPreparing(pyfile) + self.core.hookManager.downloadPreparing(pyfile) pyfile.plugin.preprocessing(self) - self.m.log.info(_("Download finished: %s") % pyfile.name) - self.m.core.hookManager.downloadFinished(pyfile) - self.m.core.files.checkPackageFinished(pyfile) + self.log.info(_("Download finished: %s") % pyfile.name) + self.core.hookManager.downloadFinished(pyfile) + self.core.files.checkPackageFinished(pyfile) except NotImplementedError: - self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + self.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) pyfile.setStatus("failed") pyfile.error = "Plugin does not work" self.clean(pyfile) @@ -79,7 +79,7 @@ class DownloadThread(BaseThread): except Abort: try: - self.m.log.info(_("Download aborted: %s") % pyfile.name) + self.log.info(_("Download aborted: %s") % pyfile.name) except: pass @@ -99,7 +99,7 @@ class DownloadThread(BaseThread): except Retry, e: reason = e.args[0] - self.m.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) + self.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) self.queue.put(pyfile) continue @@ -108,16 +108,16 @@ class DownloadThread(BaseThread): if msg == "offline": pyfile.setStatus("offline") - self.m.log.warning(_("Download is offline: %s") % pyfile.name) + self.log.warning(_("Download is offline: %s") % pyfile.name) elif msg == "temp. offline": pyfile.setStatus("temp. offline") - self.m.log.warning(_("Download is temporary offline: %s") % pyfile.name) + self.log.warning(_("Download is temporary offline: %s") % pyfile.name) else: pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + self.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) pyfile.error = msg - self.m.core.hookManager.downloadFailed(pyfile) + self.core.hookManager.downloadFailed(pyfile) self.clean(pyfile) continue @@ -128,10 +128,10 @@ class DownloadThread(BaseThread): code = 0 msg = e.args - self.m.log.debug("pycurl exception %s: %s" % (code, msg)) + self.log.debug("pycurl exception %s: %s" % (code, msg)) if code in (7, 18, 28, 52, 56): - self.m.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) + self.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) wait = time() + 60 pyfile.waitUntil = wait @@ -142,7 +142,7 @@ class DownloadThread(BaseThread): break if pyfile.abort: - self.m.log.info(_("Download aborted: %s") % pyfile.name) + self.log.info(_("Download aborted: %s") % pyfile.name) pyfile.setStatus("aborted") self.clean(pyfile) @@ -153,12 +153,12 @@ class DownloadThread(BaseThread): else: pyfile.setStatus("failed") - self.m.log.error("pycurl error %s: %s" % (code, msg)) - if self.m.core.debug: + self.log.error("pycurl error %s: %s" % (code, msg)) + if self.core.debug: print_exc() self.writeDebugReport(pyfile.pluginname, pyfile) - self.m.core.hookManager.downloadFailed(pyfile) + self.core.hookManager.downloadFailed(pyfile) self.clean(pyfile) continue @@ -166,34 +166,34 @@ class DownloadThread(BaseThread): except SkipDownload, e: pyfile.setStatus("skipped") - self.m.log.info( - _("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) + self.log.info(_("Download skipped: %(name)s due to %(plugin)s") + % {"name": pyfile.name, "plugin": e.message}) self.clean(pyfile) - self.m.core.files.checkPackageFinished(pyfile) + self.core.files.checkPackageFinished(pyfile) self.active = False - self.m.core.files.save() + self.core.files.save() continue except Exception, e: pyfile.setStatus("failed") - self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + self.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) pyfile.error = str(e) - if self.m.core.debug: + if self.core.debug: print_exc() self.writeDebugReport(pyfile.pluginname, pyfile) - self.m.core.hookManager.downloadFailed(pyfile) + self.core.hookManager.downloadFailed(pyfile) self.clean(pyfile) continue finally: - self.m.core.files.save() + self.core.files.save() pyfile.checkIfProcessed() exc_clear() @@ -202,7 +202,7 @@ class DownloadThread(BaseThread): self.active = False pyfile.finishIfDone() - self.m.core.files.save() + self.core.files.save() def put(self, job): -- cgit v1.2.3 From bac28b7740aae772636d8b90e291d9c17dfd59a7 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Sun, 8 Jan 2012 14:44:59 +0100 Subject: new MultiHoster hook --- module/threads/DownloadThread.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/threads/DownloadThread.py') diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py index e140703d5..c151831a3 100644 --- a/module/threads/DownloadThread.py +++ b/module/threads/DownloadThread.py @@ -156,7 +156,7 @@ class DownloadThread(BaseThread): self.log.error("pycurl error %s: %s" % (code, msg)) if self.core.debug: print_exc() - self.writeDebugReport(pyfile.pluginname, pyfile) + self.writeDebugReport(pyfile.plugin.__name__, pyfile) self.core.hookManager.downloadFailed(pyfile) @@ -186,7 +186,7 @@ class DownloadThread(BaseThread): if self.core.debug: print_exc() - self.writeDebugReport(pyfile.pluginname, pyfile) + self.writeDebugReport(pyfile.plugin.__name__, pyfile) self.core.hookManager.downloadFailed(pyfile) self.clean(pyfile) -- cgit v1.2.3 From ebe0e6039d822e9c16a6095dba8691066bc3b466 Mon Sep 17 00:00:00 2001 From: Pedro Algarvio Date: Mon, 13 Feb 2012 12:56:40 +0000 Subject: Catch internal server errors on the right place. --- module/threads/DownloadThread.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'module/threads/DownloadThread.py') diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py index c151831a3..879dbf8bd 100644 --- a/module/threads/DownloadThread.py +++ b/module/threads/DownloadThread.py @@ -26,6 +26,7 @@ from pycurl import error from module.plugins.Base import Fail, Retry from module.plugins.Hoster import Abort, Reconnect, SkipDownload +from module.network.HTTPRequest import BadHeader from BaseThread import BaseThread @@ -102,7 +103,12 @@ class DownloadThread(BaseThread): self.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) self.queue.put(pyfile) continue - + except BadHeader, e: + if e.code == 500: + self.log.info(_("Internal Server Error")) + pyfile.error = _("Internal Server Error") + pyfile.plugin.tempOffline() + raise e except Fail, e: msg = e.args[0] @@ -212,4 +218,4 @@ class DownloadThread(BaseThread): def stop(self): """stops the thread""" - self.put("quit") \ No newline at end of file + self.put("quit") -- cgit v1.2.3 From 224683926624cf05d3441dae157de1a0ab68b973 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 13 Feb 2012 14:14:38 +0100 Subject: catch server errors correctly --- module/threads/DownloadThread.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'module/threads/DownloadThread.py') diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py index 879dbf8bd..bd15b9b87 100644 --- a/module/threads/DownloadThread.py +++ b/module/threads/DownloadThread.py @@ -103,12 +103,6 @@ class DownloadThread(BaseThread): self.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) self.queue.put(pyfile) continue - except BadHeader, e: - if e.code == 500: - self.log.info(_("Internal Server Error")) - pyfile.error = _("Internal Server Error") - pyfile.plugin.tempOffline() - raise e except Fail, e: msg = e.args[0] @@ -185,10 +179,16 @@ class DownloadThread(BaseThread): continue - except Exception, e: - pyfile.setStatus("failed") - self.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) - pyfile.error = str(e) + except (Exception, BadHeader), e: + if isinstance(e, BadHeader) and e.code == 500: + pyfile.setStatus("temp. offline") + self.log.warning(_("Download is temporary offline: %s") % pyfile.name) + pyfile.error = _("Internal Server Error") + + else: + pyfile.setStatus("failed") + self.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) if self.core.debug: print_exc() -- cgit v1.2.3 From 4df2b77fdf42046fe19bd371be7c7255986b5980 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 6 Mar 2012 13:36:39 +0100 Subject: renamed hooks to addons, new filemanager and database, many new api methods you will loose ALL your LINKS, webinterface will NOT work --- module/threads/DownloadThread.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'module/threads/DownloadThread.py') diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py index bd15b9b87..6239cddd8 100644 --- a/module/threads/DownloadThread.py +++ b/module/threads/DownloadThread.py @@ -64,11 +64,11 @@ class DownloadThread(BaseThread): self.log.info(_("Download starts: %s" % pyfile.name)) # start download - self.core.hookManager.downloadPreparing(pyfile) + self.core.addonManager.downloadPreparing(pyfile) pyfile.plugin.preprocessing(self) self.log.info(_("Download finished: %s") % pyfile.name) - self.core.hookManager.downloadFinished(pyfile) + self.core.addonManager.downloadFinished(pyfile) self.core.files.checkPackageFinished(pyfile) except NotImplementedError: @@ -117,7 +117,7 @@ class DownloadThread(BaseThread): self.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) pyfile.error = msg - self.core.hookManager.downloadFailed(pyfile) + self.core.addonManager.downloadFailed(pyfile) self.clean(pyfile) continue @@ -158,7 +158,7 @@ class DownloadThread(BaseThread): print_exc() self.writeDebugReport(pyfile.plugin.__name__, pyfile) - self.core.hookManager.downloadFailed(pyfile) + self.core.addonManager.downloadFailed(pyfile) self.clean(pyfile) continue @@ -179,7 +179,7 @@ class DownloadThread(BaseThread): continue - except (Exception, BadHeader), e: + except Exception, e: if isinstance(e, BadHeader) and e.code == 500: pyfile.setStatus("temp. offline") self.log.warning(_("Download is temporary offline: %s") % pyfile.name) @@ -194,7 +194,7 @@ class DownloadThread(BaseThread): print_exc() self.writeDebugReport(pyfile.plugin.__name__, pyfile) - self.core.hookManager.downloadFailed(pyfile) + self.core.addonManager.downloadFailed(pyfile) self.clean(pyfile) continue -- cgit v1.2.3 From 50d4df8b4d48b855bd18e9922355b7f3f2b4da4e Mon Sep 17 00:00:00 2001 From: RaNaN Date: Tue, 20 Mar 2012 14:57:45 +0100 Subject: captcha decrypting for all plugin types, new interaction manager --- module/threads/DownloadThread.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/threads/DownloadThread.py') diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py index 6239cddd8..8166191af 100644 --- a/module/threads/DownloadThread.py +++ b/module/threads/DownloadThread.py @@ -24,8 +24,8 @@ from traceback import print_exc from sys import exc_clear from pycurl import error -from module.plugins.Base import Fail, Retry -from module.plugins.Hoster import Abort, Reconnect, SkipDownload +from module.plugins.Base import Fail, Retry, Abort +from module.plugins.Hoster import Reconnect, SkipDownload from module.network.HTTPRequest import BadHeader from BaseThread import BaseThread -- cgit v1.2.3 From b40b32ee05f611323a7827fad2a25fa0a28dcb24 Mon Sep 17 00:00:00 2001 From: X3n0m0rph59 Date: Sun, 22 Apr 2012 19:56:17 +0200 Subject: a huge pile of spelling fixes --- module/threads/DownloadThread.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'module/threads/DownloadThread.py') diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py index 8166191af..7555a82ce 100644 --- a/module/threads/DownloadThread.py +++ b/module/threads/DownloadThread.py @@ -58,7 +58,7 @@ class DownloadThread(BaseThread): try: if not pyfile.hasPlugin(): continue - #this pyfile was deleted while queueing + #this pyfile was deleted while queuing pyfile.plugin.checkForSameFiles(starting=True) self.log.info(_("Download starts: %s" % pyfile.name)) @@ -212,7 +212,7 @@ class DownloadThread(BaseThread): def put(self, job): - """assing job to thread""" + """assign a job to the thread""" self.queue.put(job) -- cgit v1.2.3 From 1a55cb6a2eb8784253410b2e93510b5bcebf7f41 Mon Sep 17 00:00:00 2001 From: RaNaN Date: Mon, 10 Sep 2012 15:12:55 +0200 Subject: userApi for plugins --- module/threads/DownloadThread.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'module/threads/DownloadThread.py') diff --git a/module/threads/DownloadThread.py b/module/threads/DownloadThread.py index 7555a82ce..0269b0660 100644 --- a/module/threads/DownloadThread.py +++ b/module/threads/DownloadThread.py @@ -106,6 +106,8 @@ class DownloadThread(BaseThread): except Fail, e: msg = e.args[0] + # TODO: activate former skipped downloads + if msg == "offline": pyfile.setStatus("offline") self.log.warning(_("Download is offline: %s") % pyfile.name) -- cgit v1.2.3