#!/usr/bin/env python # -*- coding: utf-8 -*- """ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, see . @author: spoob @author: sebnapi @author: RaNaN @author: mkaay @version: v0.3.2 """ CURRENT_VERSION = '0.3.2' from copy import deepcopy from getopt import GetoptError from getopt import getopt import gettext from glob import glob from imp import find_module import logging import logging.handlers from operator import attrgetter from os import _exit from os import chdir from os import execv from os import makedirs from os import name as platform from os import remove from os import sep from os.path import abspath from os.path import basename from os.path import dirname from os.path import exists from os.path import isabs from os.path import join from os.path import expanduser from re import sub import signal import subprocess import sys from sys import argv from sys import executable from sys import exit from sys import path from sys import stdout from sys import version_info from tempfile import NamedTemporaryFile import thread import time from time import sleep from xmlrpclib import Binary from module.XMLConfigParser import XMLConfigParser from module.network.Request import getURL import module.remote.SecureXMLRPCServer as Server from module.web.ServerThread import WebServer from module.ThreadManager import ThreadManager from module.CaptchaManager import CaptchaManager from module.HookManager import HookManager from module.PullEvents import PullManager from module.PluginManager import PluginManager from module.FileList import FileList class Core(object): """ pyLoad Core """ def __init__(self): self.doDebug = False self.arg_links = [] self.path = abspath(dirname(__file__)) chdir(self.path) self.homedir = self.getHomeDir() if len(argv) > 1: try: options, args = getopt(argv[1:], 'vca:hdusC:', ["version", "clear", "add=", "help", "debug", "user", "setup", "configdir="]) customConfig = None for option, argument in options: if option in ("-C", "--configdir"): customConfig = argument if customConfig: self.configdir = self.make_path(customConfig) else: self.defaultConfig() self.initConfig() for option, argument in options: if option in ("-v", "--version"): print "pyLoad", CURRENT_VERSION exit() elif option in ("-c", "--clear"): try: remove(join(self.configdir, "module", "links.pkl")) print "Removed Linklist" except: print "No Linklist found" elif option in ("-a", "--add"): self.arg_links.append(argument) print "Added %s" % argument elif option in ("-h", "--help"): self.print_help() exit() elif option in ("-d", "--debug"): self.doDebug = True elif option in ("-u", "--user"): from module.setup import Setup self.xmlconfig = XMLConfigParser(self.make_path(self.configdir, "core.xml"), defaultFile=join(self.path, "module", "config", "core_default.xml")) self.config = self.xmlconfig.getConfig() s = Setup(self.path, self.config) s.set_user() exit() elif option in ("-s", "--setup"): from module.setup import Setup self.xmlconfig = XMLConfigParser(self.make_path(self.configdir, "core.xml"), defaultFile=join(self.path, "module", "config", "core_default.xml")) self.config = self.xmlconfig.getConfig() s = Setup(self.path, self.config) s.start() exit() except GetoptError: print 'Unknown Argument(s) "%s"' % " ".join(argv[1:]) self.print_help() exit() else: self.defaultConfig() self.initConfig() def defaultConfig(self): if platform == "posix": self.configdir = self.make_path(self.homedir, ".config", "pyload") else: self.configdir = self.make_path(self.homedir, "pyload") self.check_file(self.configdir, "folder for config files", True, quiet=True) def initConfig(self): #check if no config exists, assume its first run if not exists(join(self.configdir, "core.xml")): print "No configuration file found." print "Startig Configuration Assistent" print "" from module.setup import Setup self.xmlconfig = XMLConfigParser(self.make_path(self.configdir, "core.xml"), defaultFile=join(self.path, "module", "config", "core_default.xml")) self.config = self.xmlconfig.getConfig() s = Setup(self.path, self.config) try: result = s.start() if not result: remove(join(self.configdir, "core.xml")) except Exception, e: print e remove(join(self.configdir, "core.xml")) exit() def print_help(self): print "" print "pyLoad %s Copyright (c) 2008-2010 the pyLoad Team" % CURRENT_VERSION print "" print "Usage: [python] pyLoadCore.py [options]" print "" print "" print " -v, --version", " " * 10, "Print version to terminal" print " -c, --clear", " " * 12, "Delete the saved linklist" print " -a, --add=", " " * 2, "Add the specified links" print " -u, --user", " " * 13, "Set new User and password" print " -d, --debug", " " * 12, "Enable debug mode" print " -s, --setup", " " * 12, "Run Setup Assistent" print " -C, --configdir=", " " * 1, "Custom config dir (includes db)" print " -h, --help", " " * 13, "Display this help screen" print "" def toggle_pause(self): if self.thread_list.pause: self.thread_list.pause = False return False elif not self.thread_list.pause: self.thread_list.pause = True return True def quit(self, a, b): self.shutdown() self.logger.info(_("Received Quit signal")) _exit(1) def start(self): """ starts the machine""" try: signal.signal(signal.SIGQUIT, self.quit) except: pass self.config = {} self.plugins_avaible = {} self.plugin_folder = self.make_path("module", "plugins") self.xmlconfig = XMLConfigParser(self.make_path(self.configdir, "core.xml"), defaultFile=self.make_path(self.path, "module", "config", "core_default.xml")) self.config = self.xmlconfig.getConfig() if self.doDebug == True: self.config['general']['debug_mode'] = True self.parser_plugins = XMLConfigParser(self.make_path(self.configdir, "plugin.xml"), defaultFile=self.make_path(self.path, "module", "config", "plugin_default.xml")) self.config['ssl']['cert'] = self.make_path(self.config['ssl']['cert']) self.config['ssl']['key'] = self.make_path(self.config['ssl']['key']) self.do_kill = False self.do_restart = False translation = gettext.translation("pyLoad", self.make_path("locale"), languages=[self.config['general']['language']]) translation.install(unicode=(True if sys.getfilesystemencoding().lower().startswith("utf") else False)) self.check_install("Crypto", _("pycrypto to decode container files")) self.check_install("Image", _("Python Image Libary (PIL) for captha reading")) self.check_install("pycurl", _("pycurl to download any files"), True, True) self.check_install("django", _("Django for webinterface")) self.check_install("tesseract", _("tesseract for captcha reading"), False) self.check_install("gocr", _("gocr for captcha reading"), False) self.check_file(self.make_path(self.config['log']['log_folder']), _("folder for logs"), True) self.check_file(self.make_path(self.config['general']['download_folder']), _("folder for downloads"), True) self.check_file(self.make_path(self.config['general']['link_file']), _("file for links")) self.check_file(self.make_path(self.config['general']['failed_file']), _("file for failed links")) if self.config['ssl']['activated']: self.check_install("OpenSSL", _("OpenSSL for secure connection"), True) self.check_file(self.make_path(self.config['ssl']['cert']), _("ssl certificate"), False, True) self.check_file(self.make_path(self.config['ssl']['key']), _("ssl key"), False, True) self.downloadSpeedLimit = int(self.xmlconfig.get("general", "download_speed_limit", 0)) if self.config['general']['debug_mode']: self.init_logger(logging.DEBUG) # logging level else: self.init_logger(logging.INFO) # logging level self.init_hooks() path.append(self.plugin_folder) self.create_plugin_index() self.lastGuiConnected = 0 self.server_methods = ServerMethods(self) self.file_list = FileList(self) self.pullManager = PullManager(self) self.thread_list = ThreadManager(self) self.captchaManager = CaptchaManager(self) self.last_update_check = 0 self.update_check_interval = 6 * 60 * 60 self.update_available = self.check_update() self.logger.info(_("Downloadtime: %s") % self.server_methods.is_time_download()) self.init_server() self.init_webserver() linkFile = self.config['general']['link_file'] packs = self.server_methods.get_queue() found = False for data in packs: if data["package_name"] == linkFile: found = data["id"] break if found == False: pid = self.file_list.packager.addNewPackage(package_name=linkFile) else: pid = found lid = self.file_list.collector.addLink(linkFile) try: self.file_list.packager.addFileToPackage(pid, self.file_list.collector.popFile(lid)) if self.arg_links: for link in self.arg_links: lid = self.file_list.collector.addLink(link) self.file_list.packager.addFileToPackage(pid, self.file_list.collector.popFile(lid)) self.file_list.packager.pushPackage2Queue(pid) self.file_list.continueAborted() except: pass freeSpace = self.freeSpace() if freeSpace > 5 * 1024: self.logger.info(_("Free space: %sGB") % (freeSpace / 1024)) else: self.logger.info(_("Free space: %sMB") % freeSpace) self.thread_list.pause = False self.thread_list.start() self.hookManager.coreReady() while True: sleep(2) if self.do_restart: self.logger.info(_("restarting pyLoad")) self.restart() if self.do_kill: self.shutdown() self.logger.info(_("pyLoad quits")) exit() if self.last_update_check + self.update_check_interval <= time.time(): self.update_available = self.check_update() def init_server(self): try: server_addr = (self.config['remote']['listenaddr'], int(self.config['remote']['port'])) usermap = {self.config['remote']['username']: self.config['remote']['password']} if self.config['ssl']['activated']: self.server = Server.SecureXMLRPCServer(server_addr, self.config['ssl']['cert'], self.config['ssl']['key'], usermap) self.logger.info(_("Secure XMLRPC Server Started")) else: self.server = Server.AuthXMLRPCServer(server_addr, usermap) self.logger.info(_("Auth XMLRPC Server Started")) self.server.register_instance(self.server_methods) thread.start_new_thread(self.server.serve_forever, ()) except Exception, e: self.logger.error(_("Failed starting XMLRPC server CLI and GUI will not be available: %s") % str(e)) if self.config['general']['debug_mode']: import traceback traceback.print_exc() def init_webserver(self): if self.config['webinterface']['activated']: self.webserver = WebServer(self) self.webserver.start() def init_logger(self, level): console = logging.StreamHandler(stdout) frm = logging.Formatter("%(asctime)s: %(levelname)-8s %(message)s", "%d.%m.%Y %H:%M:%S") console.setFormatter(frm) self.logger = logging.getLogger("log") # settable in config if self.config['log']['file_log']: file_handler = logging.handlers.RotatingFileHandler(join(self.path, self.config['log']['log_folder'], 'log.txt'), maxBytes=102400, backupCount=int(self.config['log']['log_count'])) #100 kib each file_handler.setFormatter(frm) self.logger.addHandler(file_handler) self.logger.addHandler(console) #if console logging self.logger.setLevel(level) def init_hooks(self): self.hookManager = HookManager(self) def check_install(self, check_name, legend, python=True, essential=False): """check wether needed tools are installed""" try: if python: find_module(check_name) else: pipe = subprocess.PIPE subprocess.Popen(check_name, stdout=pipe, stderr=pipe) except: print _("Install %s") % legend if essential: exit() def check_file(self, check_names, description="", folder=False, empty=True, essential=False, quiet=False): """check wether needed files exists""" tmp_names = [] if not type(check_names) == list: tmp_names.append(check_names) else: tmp_names.extend(check_names) file_created = True file_exists = True for tmp_name in tmp_names: if not exists(tmp_name): file_exists = False if empty: try: if folder: tmp_name = tmp_name.replace("/", sep) makedirs(tmp_name) else: open(tmp_name, "w") except: file_created = False else: file_created = False if not file_exists and not quiet: if file_created: print _("%s created") % description else: if not empty: print _("could not find %s: %s") % (description, tmp_name) else: print _("could not create %s: %s") % (description, tmp_name) if essential: exit() def isGUIConnected(self): return self.lastGuiConnected + 10 > time.time() def restart(self): self.shutdown() execv(executable, [executable, "pyLoadCore.py"]) def create_plugin_index(self): self.pluginManager = PluginManager(self) def compare_time(self, start, end): start = map(int, start) end = map(int, end) if start == end: return True now = list(time.localtime()[3:5]) if start < now and end > now: return True elif start > end and (now > start or now < end): return True elif start < now and end < now and start > end: return True else: return False def getMaxSpeed(self): return self.downloadSpeedLimit def shutdown(self): self.logger.info(_("shutting down...")) try: if self.config['webinterface']['activated']: self.webserver.quit() #self.webserver.join() for thread in self.thread_list.threads: thread.shutdown = True self.thread_list.stopAllDownloads() for thread in self.thread_list.threads: thread.join(10) self.file_list.save() except: self.logger.info(_("error while shutting down")) def check_update(self): try: if self.config['updates']['search_updates']: version_check = getURL("http://get.pyload.org/check/%s/" % (CURRENT_VERSION,)) if version_check == "": self.logger.info(_("No Updates for pyLoad")) return False else: self.logger.info(_("New pyLoad Version %s available") % version_check) return True else: return False except: self.logger.error(_("Not able to connect server")) finally: self.last_update_check = time.time() def install_update(self): try: if self.config['updates']['search_updates']: if self.core.config['updates']['install_updates']: version_check = getURL("http://get.pyload.org/get/update/%s/" % (CURRENT_VERSION,)) else: version_check = getURL("http://get.pyload.org/check/%s/" % (CURRENT_VERSION,)) if version_check == "": return False else: if self.config['updates']['install_updates']: try: tmp_zip_name = __import__("tempfile").NamedTemporaryFile(suffix=".zip").name tmp_zip = open(tmp_zip_name, 'wb') tmp_zip.write(version_check) tmp_zip.close() __import__("module.Unzip", globals(), locals(), "Unzip", -1).Unzip().extract(tmp_zip_name, "Test/") return True except: self.logger.info(_("Auto install Failed")) return False else: return False else: return False finally: return False def make_path(self, *args): if isabs(args[0]): return join(*args) else: return join(self.path, *args) def freeSpace(self): folder = self.make_path(self.config['general']['download_folder']) if platform == 'nt': import ctypes free_bytes = ctypes.c_ulonglong(0) ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(folder), None, None, ctypes.pointer(free_bytes)) return free_bytes.value / 1024 / 1024 #megabyte else: from os import statvfs s = statvfs(folder) return s.f_bsize * s.f_bavail / 1024 / 1024 #megabyte def getHomeDir(self): try: from win32com.shell import shellcon, shell return shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0) except ImportError: # quick semi-nasty fallback for non-windows/win32com case return expanduser("~") #################################### ########## XMLRPC Methods ########## #################################### class ServerMethods(): def __init__(self, core): self.core = core def status_downloads(self): downloads = [] for pyfile in self.core.thread_list.py_downloading: download = {} download['id'] = pyfile.id download['name'] = pyfile.status.filename download['speed'] = pyfile.status.get_speed() download['eta'] = pyfile.status.get_ETA() download['kbleft'] = pyfile.status.kB_left() download['size'] = pyfile.status.size() download['percent'] = pyfile.status.percent() download['status'] = pyfile.status.type download['wait_until'] = pyfile.status.waituntil download['package'] = pyfile.package.data["package_name"] downloads.append(download) return downloads def get_conf_val(self, cat, var): if var != "username" and var != "password": return self.core.config[cat][var] else: raise Exception("not allowed!") def set_conf_val(self, cat, opt, val): if opt not in ("username", "password"): self.core.config[str(cat)][str(opt)] = val else: raise Exception("not allowed!") def get_config(self): d = deepcopy(self.core.xmlconfig.getConfigDict()) del d["remote"]["username"] del d["remote"]["password"] return d def get_config_data(self): d = deepcopy(self.core.xmlconfig.getDataDict()) del d["remote"]["options"]["username"] del d["remote"]["options"]["password"] return d def pause_server(self): self.core.thread_list.pause = True def unpause_server(self): self.core.thread_list.pause = False def toggle_pause(self): if self.core.thread_list.pause: self.core.thread_list.pause = False else: self.core.thread_list.pause = True return self.core.thread_list.pause def status_server(self): status = {} status['pause'] = self.core.thread_list.pause status['activ'] = len(self.core.thread_list.py_downloading) status['queue'] = self.core.file_list.countDownloads() status['total'] = len(self.core.file_list.data['queue']) status['speed'] = 0 for pyfile in self.core.thread_list.py_downloading: status['speed'] += pyfile.status.get_speed() status['download'] = not self.core.thread_list.pause and self.is_time_download() status['reconnect'] = self.core.config['reconnect']['activated'] and self.is_time_reconnect() return status def file_exists(self, path): #@XXX: security?! return exists(path) def get_server_version(self): return CURRENT_VERSION def add_urls(self, links): for link in links: link = link.strip() if link.startswith("http") or exists(link): self.core.file_list.collector.addLink(link) self.core.file_list.save() def add_package(self, name, links, queue=True): pid = self.new_package(name) fids = map(self.core.file_list.collector.addLink, links) map(lambda fid: self.move_file_2_package(fid, pid), fids) if queue: self.core.file_list.packager.pushPackage2Queue(pid) self.core.file_list.save() def new_package(self, name): id = self.core.file_list.packager.addNewPackage(name) self.core.file_list.save() return id def get_package_data(self, id): return self.core.file_list.packager.getPackageData(id) def get_package_files(self, id): return self.core.file_list.packager.getPackageFiles(id) def get_file_info(self, id): return self.core.file_list.getFileInfo(id) def del_links(self, ids): for id in ids: try: self.core.file_list.collector.removeFile(id) except: self.core.file_list.packager.removeFile(id) self.core.file_list.save() def del_packages(self, ids): map(self.core.file_list.packager.removePackage, ids) self.core.file_list.save() def kill(self): self.core.do_kill = True return True def restart(self): self.core.do_restart = True def get_queue(self): return map(attrgetter("data"), self.core.file_list.data["queue"]) def get_collector_packages(self): return map(attrgetter("data"), self.core.file_list.data["packages"]) def get_collector_files(self): return map(attrgetter("id"), self.core.file_list.data["collector"]) def move_file_2_package(self, fid, pid): try: pyfile = self.core.file_list.collector.getFile(fid) self.core.file_list.packager.addFileToPackage(pid, pyfile) except: return else: self.core.file_list.collector.removeFile(fid) def push_package_2_queue(self, id): self.core.file_list.packager.pushPackage2Queue(id) def restart_package(self, packid): map(self.core.file_list.packager.resetFileStatus, self.core.file_list.packager.getPackageFiles(packid)) def restart_file(self, fileid): self.core.file_list.packager.resetFileStatus(fileid) def upload_container(self, filename, type, content): th = NamedTemporaryFile(mode="w", suffix="." + type, delete=False) th.write(content) path = th.name th.close() pid = self.core.file_list.packager.addNewPackage(filename) cid = self.core.file_list.collector.addLink(path) self.move_file_2_package(cid, pid) self.core.file_list.save() def get_log(self, offset=0): filename = self.core.config['log']['log_folder'] + sep + 'log.txt' fh = open(filename, "r") content = fh.read() fh.close() lines = content.splitlines() if offset >= len(lines): return None return lines[offset:] def stop_downloads(self): self.core.thread_list.stopAllDownloads() def stop_download(self, type, id): if type == "pack": ids = self.core.file_list.getPackageFiles(id) for fid in ids: self.core.file_list.packager.abortFile(fid) else: self.core.file_list.packager.abortFile(id) def update_available(self): return self.core.update_available def set_package_name(self, pid, name): self.core.file_list.packager.setPackageData(pid, package_name=name) def pull_out_package(self, pid): self.core.file_list.packager.pullOutPackage(pid) def is_captcha_waiting(self): self.core.lastGuiConnected = time.time() task = self.core.captchaManager.getTask() return not task == None def get_captcha_task(self, exclusive=False): self.core.lastGuiConnected = time.time() task = self.core.captchaManager.getTask() if task: task.setWatingForUser(exclusive=exclusive) c = task.getCaptcha() return str(task.getID()), Binary(c[0]), str(c[1]) else: return None, None, None def get_task_status(self, tid): self.core.lastGuiConnected = time.time() return self.core.captchaManager.getTaskFromID(tid).getStatus() def set_captcha_result(self, tid, result): self.core.lastGuiConnected = time.time() task = self.core.captchaManager.getTaskFromID(tid) if task: task.setResult(result) task.setDone() return True else: return False def get_events(self, uuid): return self.core.pullManager.getEvents(uuid) def get_full_queue(self): data = [] for pack in self.core.file_list.data["queue"]: p = {"data":pack.data, "children":[]} for child in pack.files: info = self.core.file_list.getFileInfo(child.id) info["downloading"] = None p["children"].append(info) data.append(p) return data #def move_urls_up(self, ids): # for id in ids: # self.core.file_list.move(id) # self.core.file_list.save() #def move_urls_down(self, ids): # for id in ids: # self.core.file_list.move(id, 1) # self.core.file_list.save() def is_time_download(self): start = self.core.config['downloadTime']['start'].split(":") end = self.core.config['downloadTime']['end'].split(":") return self.core.compare_time(start, end) def is_time_reconnect(self): start = self.core.config['reconnect']['startTime'].split(":") end = self.core.config['reconnect']['endTime'].split(":") return self.core.compare_time(start, end) # And so it begins... if __name__ == "__main__": pyload_core = Core() try: pyload_core.start() except KeyboardInterrupt: pyload_core.shutdown() pyload_core.logger.info(_("killed pyLoad from Terminal")) _exit(1)