#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License,
or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, see .
@author: RaNaN
"""
from Queue import Queue
from threading import Thread
from time import sleep
from time import time
from time import strftime
from traceback import print_exc, format_exc
from pprint import pformat
from sys import exc_info, exc_clear
from types import MethodType
from os.path import join, exists
from module.plugins.Plugin import Abort
from module.plugins.Plugin import Fail
from module.plugins.Plugin import Reconnect
from module.plugins.Plugin import Retry
from pycurl import error
from module.FileDatabase import PyFile
########################################################################
class PluginThread(Thread):
"""abstract base class for thread types"""
#----------------------------------------------------------------------
def __init__(self, manager):
"""Constructor"""
Thread.__init__(self)
self.setDaemon(True)
self.m = manager #thread manager
def writeDebugReport(self, pyfile):
dump = "pyLoad %s Debug Report of %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % (self.m.core.server_methods.get_server_version(), pyfile.pluginname, format_exc())
tb = exc_info()[2]
stack = []
while tb:
stack.append(tb.tb_frame)
tb = tb.tb_next
for frame in stack[1:]:
dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name,
frame.f_code.co_filename,
frame.f_lineno)
for key, value in frame.f_locals.items():
dump += "\t%20s = " % key
try:
dump += pformat(value) + "\n"
except Exception, e:
dump += " "+ str(e) +"\n"
dump += "\n\nPLUGIN OBJECT DUMP: \n\n"
for name in dir(pyfile.plugin):
attr = getattr(pyfile.plugin, name)
if not name.endswith("__") and type(attr) != MethodType:
dump += "\t%20s = " % name
try:
dump += pformat(attr) + "\n"
except Exception, e:
dump += " "+ str(e) +"\n"
dump += "\nPYFILE OBJECT DUMP: \n\n"
for name in dir(pyfile):
attr = getattr(pyfile, name)
if not name.endswith("__") and type(attr) != MethodType:
dump += "\t%20s = " % name
try:
dump += pformat(attr) + "\n"
except Exception, e:
dump += " "+ str(e) +"\n"
if self.m.core.config.plugin.has_key(pyfile.pluginname):
dump += "\n\nCONFIG: \n\n"
dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) +"\n"
dump_name = "debug_%s_%s.txt" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S"))
self.m.core.log.info("Debug Report written to %s" % dump_name)
f = open(dump_name, "wb")
f.write(dump)
f.close()
def clean(self, pyfile):
""" set thread unactive and clean pyfile """
pyfile.plugin.req.clean()
self.active = False
pyfile.release()
########################################################################
class DownloadThread(PluginThread):
"""thread for downloading files from 'real' hoster plugins"""
#----------------------------------------------------------------------
def __init__(self, manager):
"""Constructor"""
PluginThread.__init__(self, manager)
self.queue = Queue() # job queue
self.active = False
self.start()
#----------------------------------------------------------------------
def run(self):
"""run method"""
pyfile = None
while True:
del pyfile
self.active = self.queue.get()
pyfile = self.active
if self.active == "quit":
self.active = False
self.m.threads.remove(self)
return True
try:
current = False
for thread in self.m.threads:
if thread == self or not thread.active:
continue
if thread.active.name == pyfile.name and thread.active.packageid == pyfile.packageid:
current = True
if self.m.core.config["general"]["skip_existing"] and \
((not pyfile.name.startswith("http:") and exists(
join(self.m.core.config["general"]["download_folder"], pyfile.package().folder, pyfile.name)
)) or current):
self.m.log.info(_("Download skipped: %(name)s @ %(plugin)s") % {"name": pyfile.name,
"plugin": pyfile.plugin.__name__
})
pyfile.error = _("File already exists.")
else:
self.m.log.info(_("Download starts: %s" % pyfile.name))
# start download
self.m.core.hookManager.downloadStarts(pyfile)
pyfile.plugin.preprocessing(self)
except NotImplementedError:
self.m.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname)
pyfile.setStatus("failed")
pyfile.error = "Plugin does not work"
self.clean(pyfile)
continue
except Abort:
self.m.log.info(_("Download aborted: %s") % pyfile.name)
pyfile.setStatus("aborted")
self.clean(pyfile)
continue
except Reconnect:
self.queue.put(pyfile)
#pyfile.req.clearCookies()
while self.m.reconnecting.isSet():
sleep(0.5)
continue
except Retry:
self.m.log.info(_("Download restarted: %s") % pyfile.name)
self.queue.put(pyfile)
continue
except Fail, e:
msg = e.args[0]
if msg == "offline":
pyfile.setStatus("offline")
self.m.log.warning(_("Download is offline: %s") % pyfile.name)
else:
pyfile.setStatus("failed")
self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg})
pyfile.error = msg
self.clean(pyfile)
continue
except error, e:
try:
code, msg = e.args
except:
code = 0
msg = e.args
if code in (7, 18, 28, 52, 56):
self.m.log.warning(_("Couldn't connect to host or connection resetted waiting 1 minute and retry."))
wait = time() + 60
while time() < wait:
sleep(1)
if pyfile.abort:
break
if pyfile.abort:
self.m.log.info(_("Download aborted: %s") % pyfile.name)
pyfile.setStatus("aborted")
self.clean(pyfile)
else:
pyfile.plugin.req.canContinue = False
self.queue.put(pyfile)
continue
else:
pyfile.setStatus("failed")
self.m.log.error("pycurl error %s: %s" % (code, msg))
if self.m.core.debug:
print_exc()
self.writeDebugReport(pyfile)
self.clean(pyfile)
continue
except Exception, e:
pyfile.setStatus("failed")
self.m.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)})
pyfile.error = str(e)
if self.m.core.debug:
print_exc()
self.writeDebugReport(pyfile)
self.clean(pyfile)
continue
finally:
self.m.core.files.save()
exc_clear()
self.m.log.info(_("Download finished: %s") % pyfile.name)
pyfile.plugin.req.clean()
self.m.core.hookManager.downloadFinished(pyfile)
self.m.core.files.checkPackageFinished(pyfile)
self.active = False
pyfile.finishIfDone()
self.m.core.files.save()
#----------------------------------------------------------------------
def put(self, job):
"""assing job to thread"""
self.queue.put(job)
#----------------------------------------------------------------------
def stop(self):
"""stops the thread"""
self.put("quit")
########################################################################
class DecrypterThread(PluginThread):
"""thread for decrypting"""
#----------------------------------------------------------------------
def __init__(self, manager, pyfile):
"""constructor"""
PluginThread.__init__(self, manager)
self.active = pyfile
manager.localThreads.append(self)
pyfile.setStatus("decrypting")
self.start()
#----------------------------------------------------------------------
def run(self):
"""run method"""
pyfile = self.active
retry = False
try:
self.m.log.info(_("Decrypting starts: %s") % self.active.name)
self.active.plugin.preprocessing(self)
except NotImplementedError:
self.m.log.error(_("Plugin %s is missing a function.") % self.active.pluginname)
return
except Fail, e:
msg = e.args[0]
if msg == "offline":
self.active.setStatus("offline")
self.m.log.warning(_("Download is offline: %s") % self.active.name)
else:
self.active.setStatus("failed")
self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % { "name" : self.active.name, "msg":msg })
self.active.error = msg
return
except Abort:
self.m.log.info(_("Download aborted: %s") % pyfile.name)
pyfile.setStatus("aborted")
return
except Retry:
self.m.log.info(_("Retrying %s") % self.active.name)
retry = True
return self.run()
except Exception, e:
self.active.setStatus("failed")
self.m.log.error(_("Decrypting failed: %(name)s | %(msg)s") % { "name" : self.active.name, "msg" :str(e) })
self.active.error = str(e)
if self.m.core.debug:
print_exc()
self.writeDebugReport(pyfile)
return
finally:
if not retry:
self.active.release()
self.active = False
self.m.core.files.save()
self.m.localThreads.remove(self)
exc_clear()
#self.m.core.hookManager.downloadFinished(pyfile)
#self.m.localThreads.remove(self)
#self.active.finishIfDone()
if not retry:
pyfile.delete()
########################################################################
class HookThread(PluginThread):
"""thread for hooks"""
#----------------------------------------------------------------------
def __init__(self, m, function, pyfile):
"""Constructor"""
PluginThread.__init__(self, m)
self.f = function
self.active = pyfile
m.localThreads.append(self)
if isinstance(pyfile, PyFile):
pyfile.setStatus("processing")
self.start()
def run(self):
self.f(self.active)
self.m.localThreads.remove(self)
if isinstance(self.active, PyFile):
self.active.finishIfDone()
########################################################################
class InfoThread(PluginThread):
#----------------------------------------------------------------------
def __init__(self, manager, data, pid):
"""Constructor"""
PluginThread.__init__(self, manager)
self.data = data
self.pid = pid # package id
# [ .. (name, plugin) .. ]
self.start()
#----------------------------------------------------------------------
def run(self):
"""run method"""
plugins = {}
for url, plugin in self.data:
if plugins.has_key(plugin):
plugins[plugin].append(url)
else:
plugins[plugin] = [url]
for pluginname, urls in plugins.iteritems():
plugin = self.m.core.pluginManager.getPlugin(pluginname)
if hasattr(plugin, "getInfo"):
try:
self.m.core.log.debug("Run Info Fetching for %s" % pluginname)
for result in plugin.getInfo(urls):
if not type(result) == list: result = [result]
self.m.core.files.updateFileInfo(result, self.pid)
self.m.core.log.debug("Finished Info Fetching for %s" % pluginname)
self.m.core.files.save()
except Exception, e:
self.m.core.log.debug("Info Fetching for %s failed | %s" % (pluginname,str) )