#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 3 of the License,
or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, see .
@author: spoob
@author: sebnapi
@author: RaNaN
@author: mkaay
@version: v0.4.4
"""
CURRENT_VERSION = '0.4.4'
import __builtin__
from getopt import getopt, GetoptError
import gettext
from imp import find_module
import logging
import logging.handlers
import os
from os import _exit
from os import execv
from os import getcwd
from os import makedirs
from os import name as platform
from os import remove
from os import sep
from os.path import exists, join
import signal
import subprocess
import sys
from sys import argv, executable, exit
import time
from time import sleep
from traceback import print_exc
from xmlrpclib import Binary
from module import InitHomeDir
from module.plugins.AccountManager import AccountManager
from module.CaptchaManager import CaptchaManager
from module.ConfigParser import ConfigParser
from module.HookManager import HookManager
from module.plugins.PluginManager import PluginManager
from module.PullEvents import PullManager
from module.network.RequestFactory import RequestFactory
from module.ThreadManager import ThreadManager
from module.web.ServerThread import WebServer
from module.PyFile import PyFile
from module.Scheduler import Scheduler
from module.JsEngine import JsEngine
from module.remote.RemoteManager import RemoteManager
from module.DatabaseBackend import DatabaseBackend
from module.FileDatabase import FileHandler
from module.UserDatabase import UserMethods
from codecs import getwriter
if os.name == "nt":
enc = "cp850"
else:
enc = "utf8"
sys.stdout = getwriter(enc)(sys.stdout, errors = "replace")
class Core(object):
"""pyLoad Core, one tool to rule them all... (the filehosters) :D"""
def __init__(self):
self.doDebug = False
self.daemon = False
self.arg_links = []
self.pidfile = "pyload.pid"
if len(argv) > 1:
try:
options, args = getopt(argv[1:], 'vchdusq',
["version", "clear", "help", "debug", "user", "setup", "configdir=", "changedir", "daemon", "quit", "status"])
for option, argument in options:
if option in ("-v", "--version"):
print "pyLoad", CURRENT_VERSION
exit()
elif option == "--daemon":
self.daemon = True
elif option in ("-c", "--clear"):
try:
remove("files.db")
print "Removed Linklist"
except:
print "No Linklist found"
elif option in ("-h", "--help"):
self.print_help()
exit()
elif option in ("-d", "--debug"):
self.doDebug = True
elif option in ("-u", "--user"):
from module.setup import Setup
self.config = ConfigParser()
s = Setup(pypath, self.config)
s.set_user()
exit()
elif option in ("-s", "--setup"):
from module.setup import Setup
self.config = ConfigParser()
s = Setup(pypath, self.config)
s.start()
exit()
elif option == "--changedir":
from module.setup import Setup
self.config = ConfigParser()
s = Setup(pypath, self.config)
s.conf_path(True)
exit()
elif option in ("-q", "--quit"):
self.quitInstance()
exit()
elif option == "--status":
print self.isAlreadyRunning()
exit()
except GetoptError:
print 'Unknown Argument(s) "%s"' % " ".join(argv[1:])
self.print_help()
exit()
def print_help(self):
print ""
print "pyLoad %s Copyright (c) 2008-2011 the pyLoad Team" % CURRENT_VERSION
print ""
print "Usage: [python] pyLoadCore.py [options]"
print ""
print ""
print " -v, --version", " " * 10, "Print version to terminal"
print " -c, --clear", " " * 12, "Delete the saved linklist"
#print " -a, --add=", " " * 2, "Add the specified links"
print " -u, --user", " " * 13, "Set new User and password"
print " -d, --debug", " " * 12, "Enable debug mode"
print " -s, --setup", " " * 12, "Run Setup Assistent"
print " --configdir=", " " * 6, "Run with as config directory"
print " --changedir", " "* 12, "Change config dir permanently"
print " --daemon", " " * 15, "Daemonize after start"
print " --status", " " * 15, "Display pid if running or False"
print " -q, --quit", " " * 13, "Try to quit running pyLoad"
print " -h, --help", " " * 13, "Display this help screen"
print ""
def toggle_pause(self):
if self.threadManager.pause:
self.threadManager.pause = False
return False
elif not self.threadManager.pause:
self.threadManager.pause = True
return True
def quit(self, a, b):
self.shutdown()
self.log.info(_("Received Quit signal"))
_exit(1)
def writePidFile(self):
self.deletePidFile()
pid = os.getpid()
f = open(self.pidfile, "wb")
f.write(str(pid))
f.close()
def deletePidFile(self):
if self.checkPidFile():
self.log.debug("Deleting old pidfile %s" % self.pidfile)
os.remove(self.pidfile)
def checkPidFile(self):
""" return pid as int or 0"""
if os.path.isfile(self.pidfile):
f = open(self.pidfile, "rb")
pid = f.read().strip()
f.close()
if pid:
pid = int(pid)
return pid
return 0
def isAlreadyRunning(self):
pid = self.checkPidFile()
if not pid or os.name == "nt": return False
try:
os.kill(pid, 0) # 0 - default signal (does nothing)
except:
return 0
return pid
def quitInstance(self):
if os.name == "nt":
print "Not supported on windows."
return
pid = self.isAlreadyRunning()
if not pid:
print "No pyLoad running."
try:
os.kill(pid, 3) #SIGUIT
print "pyLoad successfully stopped"
except:
print "Error quitting pyLoad"
def start(self, xmlrpc=True, web=True):
""" starts the fun :D """
if not exists("pyload.conf"):
from module.setup import Setup
print "This is your first start, running configuration assistent now."
self.config = ConfigParser()
s = Setup(pypath, self.config)
try:
res = s.start()
except:
res = False
print_exc()
print "Setup failed"
if not res:
remove("pyload.conf")
exit()
try: signal.signal(signal.SIGQUIT, self.quit)
except: pass
self.config = ConfigParser()
translation = gettext.translation("pyLoad", self.path("locale"),
languages=["en", self.config['general']['language']])
translation.install(True)
self.debug = self.doDebug or self.config['general']['debug_mode']
pid = self.isAlreadyRunning()
if pid:
print _("pyLoad already running with pid %s") % pid
exit()
if os.name != "nt" and self.config["general"]["renice"]:
os.system("renice %d %d" % (self.config["general"]["renice"], os.getpid()) )
if self.config["permission"]["change_group"]:
if os.name != "nt":
try:
from grp import getgrnam
group = getgrnam(self.config["permission"]["group"])
os.setgid(group[2])
except Exception, e:
print _("Failed changing group: %s") % e
if self.config["permission"]["change_user"]:
if os.name != "nt":
try:
from pwd import getpwnam
user = getpwnam(self.config["permission"]["user"])
os.setuid(user[2])
except Exception, e:
print _("Failed changing user: %s") % e
self.check_file(self.config['log']['log_folder'], _("folder for logs"), True)
if self.debug:
self.init_logger(logging.DEBUG) # logging level
else:
self.init_logger(logging.INFO) # logging level
self.do_kill = False
self.do_restart = False
self.shuttedDown = False
self.log.info(_("Using home directory: %s") % getcwd())
self.writePidFile()
#@TODO refractor
self.check_install("Crypto", _("pycrypto to decode container files"))
#img = self.check_install("Image", _("Python Image Libary (PIL) for captcha reading"))
#self.check_install("pycurl", _("pycurl to download any files"), True, True)
self.check_install("django", _("Django for webinterface"))
self.check_file("tmp", _("folder for temporary files"), True)
#tesser = self.check_install("tesseract", _("tesseract for captcha reading"), False) if os.name != "nt" else True
self.captcha = True # checks seems to fail, althoug tesseract is available
self.check_file(self.config['general']['download_folder'], _("folder for downloads"), True)
self.check_file("links.txt", _("file for links"))
if self.config['ssl']['activated']:
self.check_install("OpenSSL", _("OpenSSL for secure connection"), True)
self.setupDB()
if self.config.oldRemoteData:
self.log.info(_("Moving old user config to DB"))
self.db.addUser(self.config.oldRemoteData["username"], self.config.oldRemoteData["password"])
self.requestFactory = RequestFactory(self)
__builtin__.pyreq = self.requestFactory
self.lastClientConnected = 0
self.server_methods = ServerMethods(self)
self.scheduler = Scheduler(self)
#hell yeah, so many important managers :D
self.pluginManager = PluginManager(self)
self.pullManager = PullManager(self)
self.accountManager = AccountManager(self)
self.threadManager = ThreadManager(self)
self.captchaManager = CaptchaManager(self)
self.hookManager = HookManager(self)
self.remoteManager = RemoteManager(self)
self.js = JsEngine()
self.log.info(_("Downloadtime: %s") % self.server_methods.is_time_download())
if xmlrpc:
self.remoteManager.startBackends()
if web:
self.init_webserver()
#linkFile = self.config['general']['link_file']
freeSpace = self.freeSpace()
if freeSpace > 5 * 1024:
self.log.info(_("Free space: %sGB") % (freeSpace / 1024))
else:
self.log.info(_("Free space: %sMB") % freeSpace)
self.threadManager.pause = False
#self.threadManager.start()
self.hookManager.coreReady()
self.config.save() #save so config files gets filled
link_file = join(pypath, "links.txt")
if exists(link_file):
f = open(link_file, "rb")
if f.read().strip():
self.server_methods.add_package("links.txt", [link_file], 1)
f.close()
link_file = "links.txt"
if exists(link_file):
f = open(link_file, "rb")
if f.read().strip():
self.server_methods.add_package("links.txt", [link_file], 1)
f.close()
self.scheduler.addJob(0, self.accountManager.getAccountInfos)
while True:
sleep(2)
if self.do_restart:
self.log.info(_("restarting pyLoad"))
self.restart()
if self.do_kill:
self.shutdown()
self.log.info(_("pyLoad quits"))
self.removeLogger()
exit()
self.threadManager.work()
self.scheduler.work()
def setupDB(self):
self.db = DatabaseBackend(self) # the backend
self.db.setup()
self.files = FileHandler(self)
self.db.manager = self.files #ugly?
def init_webserver(self):
if self.config['webinterface']['activated']:
self.webserver = WebServer(self)
self.webserver.start()
def init_logger(self, level):
console = logging.StreamHandler(sys.stdout)
frm = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s", "%d.%m.%Y %H:%M:%S")
console.setFormatter(frm)
self.log = logging.getLogger("log") # settable in config
if self.config['log']['file_log']:
file_handler = logging.handlers.RotatingFileHandler(join(self.config['log']['log_folder'], 'log.txt'),
maxBytes=102400,
backupCount=int(self.config['log']['log_count']),
encoding="utf8"
) #100 kib each
file_handler.setFormatter(frm)
self.log.addHandler(file_handler)
self.log.addHandler(console) #if console logging
self.log.setLevel(level)
def removeLogger(self):
for h in list(self.log.handlers):
self.log.removeHandler(h)
h.close()
def check_install(self, check_name, legend, python=True, essential=False):
"""check wether needed tools are installed"""
try:
if python:
find_module(check_name)
else:
pipe = subprocess.PIPE
subprocess.Popen(check_name, stdout=pipe, stderr=pipe)
return True
except:
if essential:
self.log.info(_("Install %s") % legend)
exit()
return False
def check_file(self, check_names, description="", folder=False, empty=True, essential=False, quiet=False):
"""check wether needed files exists"""
tmp_names = []
if not type(check_names) == list:
tmp_names.append(check_names)
else:
tmp_names.extend(check_names)
file_created = True
file_exists = True
for tmp_name in tmp_names:
if not exists(tmp_name):
file_exists = False
if empty:
try:
if folder:
tmp_name = tmp_name.replace("/", sep)
makedirs(tmp_name)
else:
open(tmp_name, "w")
except:
file_created = False
else:
file_created = False
if not file_exists and not quiet:
if file_created:
#self.log.info( _("%s created") % description )
pass
else:
if not empty:
self.log.warning(_("could not find %(desc)s: %(name)s") % {"desc": description, "name": tmp_name})
else:
print _("could not create %(desc)s: %(name)s") % {"desc": description, "name": tmp_name}
if essential:
exit()
def isClientConnected(self):
return (self.lastClientConnected + 30) > time.time()
def restart(self):
self.shutdown()
execv(executable, [executable, "pyLoadCore.py"])
def compare_time(self, start, end):
start = map(int, start)
end = map(int, end)
if start == end: return True
now = list(time.localtime()[3:5])
if start < now and end > now: return True
elif start > end and (now > start or now < end): return True
elif start < now and end < now and start > end: return True
else: return False
def shutdown(self):
self.log.info(_("shutting down..."))
try:
if self.config['webinterface']['activated'] and hasattr(self, "webserver"):
self.webserver.quit()
#self.webserver.join()
for thread in self.threadManager.threads:
thread.put("quit")
pyfiles = self.files.cache.values()
for pyfile in pyfiles:
pyfile.abortDownload()
except:
if self.debug:
print_exc()
self.log.info(_("error while shutting down"))
finally:
self.files.syncSave()
self.shuttedDown = True
self.deletePidFile()
def path(self, * args):
return join(pypath, * args)
def freeSpace(self):
folder = self.config['general']['download_folder']
if platform == 'nt':
import ctypes
free_bytes = ctypes.c_ulonglong(0)
ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(folder), None, None, ctypes.pointer(free_bytes))
return free_bytes.value / 1024 / 1024 #megabyte
else:
from os import statvfs
s = statvfs(folder)
return s.f_bsize * s.f_bavail / 1024 / 1024 #megabyte
####################################
########## XMLRPC Methods ##########
####################################
class ServerMethods():
""" methods that can be used by clients with xmlrpc connection"""
def __init__(self, core):
self.core = core
def _dispatch(self, method, params):
f = getattr(self, method)
try:
return f(*params)
except:
print_exc()
raise
def status_downloads(self):
""" gives status about all files currently processed """
downloads = []
for pyfile in [x.active for x in self.core.threadManager.threads + self.core.threadManager.localThreads if
x.active and x.active != "quit"]:
if not isinstance(pyfile, PyFile):
continue
download = {}
download['id'] = pyfile.id
download['name'] = pyfile.name
download['speed'] = pyfile.getSpeed()/1024
download['eta'] = pyfile.getETA()
download['format_eta'] = pyfile.formatETA()
download['kbleft'] = pyfile.getBytesLeft() #holded for backward comp.
download['bleft'] = pyfile.getBytesLeft()
download['size'] = pyfile.getSize()
download['format_size'] = pyfile.formatSize()
download['percent'] = pyfile.getPercent()
download['status'] = pyfile.status
download['statusmsg'] = pyfile.m.statusMsg[pyfile.status]
download['format_wait'] = pyfile.formatWait()
download['wait_until'] = pyfile.waitUntil
download['package'] = pyfile.package().name
downloads.append(download)
return downloads
def get_conf_val(self, cat, var, sec="core"):
""" get config value """
if sec == "core":
return self.core.config[cat][var]
elif sec == "plugin":
return self.core.config.getPlugin(cat, var)
def set_conf_val(self, cat, opt, val, sec="core"):
""" set config value """
if sec == "core":
opt = str(opt)
self.core.config[str(cat)][opt] = val
if opt in ("limit_speed", "max_speed"): #not so nice to update the limit
self.core.requestFactory.updateBucket()
elif sec == "plugin":
self.core.config.setPlugin(cat, opt, val)
def get_config(self):
""" gets complete config """
return self.core.config.config
def get_plugin_config(self):
""" gets complete plugin config """
return self.core.config.plugin
def pause_server(self):
self.core.threadManager.pause = True
def unpause_server(self):
self.core.threadManager.pause = False
def toggle_pause(self):
if self.core.threadManager.pause:
self.core.threadManager.pause = False
else:
self.core.threadManager.pause = True
return self.core.threadManager.pause
def status_server(self):
""" dict with current server status """
status = {}
status['pause'] = self.core.threadManager.pause
status['activ'] = len(self.core.threadManager.processingIds())
status['queue'] = self.core.files.getFileCount()
status['total'] = self.core.files.getFileCount()
status['speed'] = 0
for pyfile in [x.active for x in self.core.threadManager.threads if x.active and x.active != "quit"]:
status['speed'] += pyfile.getSpeed()/1024
status['download'] = not self.core.threadManager.pause and self.is_time_download()
status['reconnect'] = self.core.config['reconnect']['activated'] and self.is_time_reconnect()
return status
def free_space(self):
return self.core.freeSpace()
def get_server_version(self):
return CURRENT_VERSION
def add_package(self, name, links, queue=0):
#0 is collector
if self.core.config['general']['folder_per_package']:
folder = name
else:
folder = ""
folder = folder.replace("http://","").replace(":","").replace("/","_").replace("\\","_")
pid = self.core.files.addPackage(name, folder, queue)
self.core.files.addLinks(links, pid)
self.core.log.info(_("Added package %(name)s containing %(count)d links") % {"name": name, "count": len(links)})
self.core.files.save()
return pid
def get_package_data(self, id):
return self.core.files.getPackageData(int(id))
def get_file_data(self, id):
info = self.core.files.getFileData(int(id))
if not info:
return None
info = {str(info.keys()[0]): info[info.keys()[0]]}
return info
def del_links(self, ids):
for id in ids:
self.core.files.deleteLink(int(id))
self.core.files.save()
def del_packages(self, ids):
for id in ids:
self.core.files.deletePackage(int(id))
self.core.files.save()
def kill(self):
self.core.do_kill = True
return True
def restart(self):
self.core.do_restart = True
def get_queue(self):
return self.core.files.getCompleteData(1)
def get_collector(self):
return self.core.files.getCompleteData(0)
def get_queue_info(self):
return self.core.files.getInfoData(1)
def get_collector_info(self):
return self.core.files.getInfoData(0)
def add_files(self, pid, links):
pid = int(pid)
self.core.files.addLinks(links, pid)
self.core.log.info(_("Added %(count)d links to package #%(package)d ") % {"count": len(links), "package": pid})
self.core.files.save()
return pid
def push_package_to_queue(self, id):
self.core.files.setPackageLocation(id, 1)
def restart_package(self, packid):
self.core.files.restartPackage(int(packid))
def recheck_package(self, packid):
self.core.files.reCheckPackage(int(packid))
def restart_file(self, fileid):
self.core.files.restartFile(int(fileid))
def upload_container(self, filename, content):
th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb")
th.write(str(content))
th.close()
self.add_package(th.name, [th.name], 1)
def get_log(self, offset=0):
filename = join(self.core.config['log']['log_folder'], 'log.txt')
try:
fh = open(filename, "r")
lines = fh.readlines()
fh.close()
if offset >= len(lines):
return None
return lines[offset:]
except:
return ('No log available', )
def stop_downloads(self):
pyfiles = self.core.files.cache.values()
for pyfile in pyfiles:
pyfile.abortDownload()
def abort_files(self, fids):
pyfiles = self.core.files.cache.values()
for pyfile in pyfiles:
if pyfile.id in fids:
pyfile.abortDownload()
def stop_download(self, type, id):
if self.core.files.cache.has_key(id):
self.core.files.cache[id].abortDownload()
def set_package_name(self, pid, name):
pack = self.core.files.getPackage(pid)
pack.name = name
pack.sync()
def pull_out_package(self, pid):
"""put package back to collector"""
self.core.files.setPackageLocation(pid, 0)
def move_package(self, dest, pid):
if dest not in (0,1): return
self.core.files.setPackageLocation(pid, dest)
def is_captcha_waiting(self):
self.core.lastClientConnected = time.time()
task = self.core.captchaManager.getTask()
return not task is None
def get_captcha_task(self, exclusive=False):
self.core.lastClientConnected = time.time()
task = self.core.captchaManager.getTask()
if task:
task.setWatingForUser(exclusive=exclusive)
c = task.getCaptcha()
return str(task.id), Binary(c[0]), str(c[1])
else:
return None, None, None
def get_task_status(self, tid):
self.core.lastClientConnected = time.time()
t = self.core.captchaManager.getTaskFromID(tid)
if t:
return t.getStatus()
def set_captcha_result(self, tid, result):
self.core.lastClientConnected = time.time()
task = self.core.captchaManager.getTaskFromID(tid)
if task:
task.setResult(result)
self.core.captchaManager.removeTask(task)
return True
else:
return False
def get_events(self, uuid):
return self.core.pullManager.getEvents(uuid)
def get_accounts(self, force=False, refresh=True):
return self.core.accountManager.getAccountInfos(force, refresh)
def update_account(self, plugin, account, password=None, options={}):
""" create and update account """
self.core.accountManager.updateAccount(plugin, account, password, options)
def remove_account(self, plugin, account):
self.core.accountManager.removeAccount(plugin, account)
def set_priority(self, id, priority):
p = self.core.files.getPackage(id)
p.setPriority(priority)
def order_package(self, id, pos):
self.core.files.reorderPackage(id, pos)
def order_file(self, id, pos):
self.core.files.reorderFile(id, pos)
def set_package_data(self, id, data):
p = self.core.files.getPackage(id)
if not p: return
for key, value in data.iteritems():
if key == "id": continue
setattr(p, key, value)
p.sync()
self.core.files.save()
def checkURLs(self, urls):
support = self.core.pluginManager.parseUrls(urls)
return [(u, p) if not p == "BasePlugin" else (u, None) for u, p in support]
def is_time_download(self):
start = self.core.config['downloadTime']['start'].split(":")
end = self.core.config['downloadTime']['end'].split(":")
return self.core.compare_time(start, end)
def is_time_reconnect(self):
start = self.core.config['reconnect']['startTime'].split(":")
end = self.core.config['reconnect']['endTime'].split(":")
return self.core.compare_time(start, end)
def delete_finished(self):
""" delete all finished links + packages, returns deleted packages """
deleted = self.core.files.deleteFinishedLinks()
return deleted
def restart_failed(self):
""" restart all failed links """
self.core.files.restartFailed()
def checkAuth(self, username, password):
return self.core.db.checkAuth(username, password)
def deamon():
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError, e:
print >>sys.stderr, "fork #1 failed: %d (%s)" % (e.errno, e.strerror)
sys.exit(1)
# decouple from parent environment
os.setsid()
os.umask(0)
# do second fork
try:
pid = os.fork()
if pid > 0:
# exit from second parent, print eventual PID before
print "Daemon PID %d" % pid
sys.exit(0)
except OSError, e:
print >>sys.stderr, "fork #2 failed: %d (%s)" % (e.errno, e.strerror)
sys.exit(1)
# Iterate through and close some file descriptors.
for fd in range(0, 3):
try:
os.close(fd)
except OSError: # ERROR, fd wasn't open to begin with (ignored)
pass
os.open(os.devnull, os.O_RDWR) # standard input (0)
os.dup2(0, 1) # standard output (1)
os.dup2(0, 2)
pyload_core = Core()
pyload_core.start()
# And so it begins...
if __name__ == "__main__":
if "--daemon" in sys.argv:
deamon()
else:
pyload_core = Core()
try:
pyload_core.start()
except KeyboardInterrupt:
pyload_core.shutdown()
pyload_core.log.info(_("killed pyLoad from Terminal"))
pyload_core.removeLogger()
_exit(1)