#!/usr/bin/env python # -*- coding: utf-8 -*- """ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, see . @author: RaNaN """ from Queue import Queue from threading import Thread from time import sleep from time import time from time import strftime from traceback import print_exc, format_exc from pprint import pformat from sys import exc_info from types import MethodType from os.path import join, exists from module.plugins.Plugin import Abort from module.plugins.Plugin import Fail from module.plugins.Plugin import Reconnect from module.plugins.Plugin import Retry from pycurl import error from module.FileDatabase import PyFile ######################################################################## class PluginThread(Thread): """abstract base class for thread types""" #---------------------------------------------------------------------- def __init__(self, manager): """Constructor""" Thread.__init__(self) self.setDaemon(True) self.m = manager #thread manager def writeDebugReport(self, pyfile): dump = "pyLoad %s Debug Report of %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % (self.m.core.server_methods.get_server_version(), pyfile.pluginname, format_exc()) tb = exc_info()[2] stack = [] while tb: stack.append(tb.tb_frame) tb = tb.tb_next for frame in stack[1:]: dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, frame.f_code.co_filename, frame.f_lineno) for key, value in frame.f_locals.items(): dump += "\t%20s = " % key try: dump += pformat(value) + "\n" except Exception, e: dump += " "+ str(e) +"\n" dump += "\n\nPLUGIN OBJECT DUMP: \n\n" for name in dir(pyfile.plugin): attr = getattr(pyfile.plugin, name) if not name.endswith("__") and type(attr) != MethodType: dump += "\t%20s = " % name try: dump += pformat(attr) + "\n" except Exception, e: dump += " "+ str(e) +"\n" dump += "\nPYFILE OBJECT DUMP: \n\n" for name in dir(pyfile): attr = getattr(pyfile, name) if not name.endswith("__") and type(attr) != MethodType: dump += "\t%20s = " % name try: dump += pformat(attr) + "\n" except Exception, e: dump += " "+ str(e) +"\n" if self.m.core.config.plugin.has_key(pyfile.pluginname): dump += "\n\nCONFIG: \n\n" dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) +"\n" dump_name = "debug_%s_%s.txt" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) self.m.core.log.info("Debug Report written to %s" % dump_name) f = open(dump_name, "wb") f.write(dump) f.close() ######################################################################## class DownloadThread(PluginThread): """thread for downloading files from 'real' hoster plugins""" #---------------------------------------------------------------------- def __init__(self, manager): """Constructor""" PluginThread.__init__(self, manager) self.queue = Queue() # job queue self.active = False self.start() #---------------------------------------------------------------------- def run(self): """run method""" while True: self.active = self.queue.get() pyfile = self.active if self.active == "quit": self.active = False self.m.threads.remove(self) return True try: current = False for thread in self.m.threads: if thread == self or not thread.active: continue if thread.active.name == pyfile.name and thread.active.packageid == pyfile.packageid: current = True if self.m.core.config["general"]["skip_existing"] and \ ((not pyfile.name.startswith("http:") and exists( join(self.m.core.config["general"]["download_folder"], pyfile.package().folder, pyfile.name) )) or current): self.m.log.info(_("Download skipped: %(name)s @ %(plugin)s") % {"name": pyfile.name, "plugin": pyfile.plugin.__name__ }) pyfile.error = _("File already exists.") else: self.m.log.info(_("Download starts: %s" % pyfile.name)) # start download self.m.core.hookManager.downloadStarts(pyfile) pyfile.plugin.preprocessing(self) except NotImplementedError: self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) pyfile.setStatus("failed") pyfile.error = "Plugin does not work" pyfile.plugin.req.clean() self.active = False pyfile.release() continue except Abort: self.m.log.info(_("Download aborted: %s") % pyfile.name) pyfile.setStatus("aborted") pyfile.plugin.req.clean() self.active = False pyfile.release() continue except Reconnect: self.queue.put(pyfile) #pyfile.req.clearCookies() while self.m.reconnecting.isSet(): sleep(0.5) continue except Retry: self.m.log.info(_("Download restarted: %s") % pyfile.name) self.queue.put(pyfile) continue except Fail, e: msg = e.args[0] if msg == "offline": pyfile.setStatus("offline") self.m.log.warning(_("Download is offline: %s") % pyfile.name) else: pyfile.setStatus("failed") self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) pyfile.error = msg pyfile.plugin.req.clean() self.active = False pyfile.release() continue except error, e: code, msg = e if code in (7,52,56,28): self.m.log.warning(_("Couldn't connect to host waiting 1 minute and retry.")) wait = time() + 60 while time() < wait: sleep(1) if pyfile.abort: break if pyfile.abort: self.m.log.info(_("Download aborted: %s") % pyfile.name) pyfile.setStatus("aborted") pyfile.plugin.req.clean() self.active = False pyfile.release() else: self.queue.put(pyfile) continue else: pyfile.setStatus("failed") self.m.log.error("pycurl error %s: %s" (code, msg)) if self.m.core.debug: print_exc() self.writeDebugReport(pyfile) pyfile.plugin.req.clean() self.active = False pyfile.release() continue except Exception, e: pyfile.setStatus("failed") self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) pyfile.error = str(e) if self.m.core.debug: print_exc() self.writeDebugReport(pyfile) pyfile.plugin.req.clean() self.active = False pyfile.release() continue finally: self.m.core.files.save() self.m.log.info(_("Download finished: %s") % pyfile.name) pyfile.plugin.req.clean() self.m.core.hookManager.downloadFinished(pyfile) self.m.core.files.checkPackageFinished(pyfile) self.active = False pyfile.finishIfDone() self.m.core.files.save() #---------------------------------------------------------------------- def put(self, job): """assing job to thread""" self.queue.put(job) #---------------------------------------------------------------------- def stop(self): """stops the thread""" self.put("quit") ######################################################################## class DecrypterThread(PluginThread): """thread for decrypting""" #---------------------------------------------------------------------- def __init__(self, manager, pyfile): """constructor""" PluginThread.__init__(self, manager) self.active = pyfile manager.localThreads.append(self) pyfile.setStatus("decrypting") self.start() #---------------------------------------------------------------------- def run(self): """run method""" pyfile = self.active try: self.m.log.info(_("Decrypting starts: %s") % self.active.name) self.active.plugin.preprocessing(self) except NotImplementedError: self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname) return except Fail, e: msg = e.args[0] if msg == "offline": self.active.setStatus("offline") self.m.log.warning(_("Download is offline: %s") % self.active.name) else: self.active.setStatus("failed") self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % { "name" : self.active.name, "msg":msg }) self.active.error = msg return except Retry: self.m.log.info(_("Retrying %s") % self.active.name) return self.active.plugin.preprocessing(self) except Exception, e: self.active.setStatus("failed") self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % { "name" : self.active.name, "msg" :str(e) }) self.active.error = str(e) if self.m.core.debug: print_exc() self.writeDebugReport(pyfile) return finally: self.active.release() self.active = False self.m.core.files.save() self.m.localThreads.remove(self) #self.m.core.hookManager.downloadFinished(pyfile) #self.m.localThreads.remove(self) #self.active.finishIfDone() pyfile.delete() ######################################################################## class HookThread(PluginThread): """thread for hooks""" #---------------------------------------------------------------------- def __init__(self, m, function, pyfile): """Constructor""" PluginThread.__init__(self, m) self.f = function self.active = pyfile m.localThreads.append(self) if isinstance(pyfile, PyFile): pyfile.setStatus("processing") self.start() def run(self): self.f(self.active) self.m.localThreads.remove(self) if isinstance(self.active, PyFile): self.active.finishIfDone() ######################################################################## class InfoThread(PluginThread): #---------------------------------------------------------------------- def __init__(self, manager, data, pid): """Constructor""" PluginThread.__init__(self, manager) self.data = data self.pid = pid # package id # [ .. (name, plugin) .. ] self.start() #---------------------------------------------------------------------- def run(self): """run method""" plugins = {} for url, plugin in self.data: if plugins.has_key(plugin): plugins[plugin].append(url) else: plugins[plugin] = [url] for pluginname, urls in plugins.iteritems(): plugin = self.m.core.pluginManager.getPlugin(pluginname) if hasattr(plugin, "getInfo"): try: self.m.core.log.debug("Run Info Fetching for %s" % pluginname) for result in plugin.getInfo(urls): if not type(result) == list: result = [result] self.m.core.files.updateFileInfo(result, self.pid) self.m.core.log.debug("Finished Info Fetching for %s" % pluginname) self.m.core.files.save() except Exception, e: self.m.core.log.debug("Info Fetching for %s failed | %s" % (pluginname,str) )