diff options
-rw-r--r-- | module/HookManager.py | 2 | ||||
-rw-r--r-- | module/file_list.py | 26 | ||||
-rw-r--r-- | module/plugins/hooks/ExternalScripts.py | 15 | ||||
-rw-r--r-- | module/plugins/hooks/LinuxFileEvents.py | 2 | ||||
-rw-r--r-- | module/web/cnl/views.py | 28 | ||||
-rw-r--r-- | module/web/urls.py | 2 | ||||
-rwxr-xr-x | pyLoadCore.py | 1280 | ||||
-rw-r--r-- | systemCheck.py | 8 |
8 files changed, 697 insertions, 666 deletions
diff --git a/module/HookManager.py b/module/HookManager.py index a283a7349..01d191ce4 100644 --- a/module/HookManager.py +++ b/module/HookManager.py @@ -28,7 +28,7 @@ from module.XMLConfigParser import XMLConfigParser class HookManager(): def __init__(self, core): self.core = core - self.configParser = XMLConfigParser(join("module", "config", "plugin.xml")) + self.configParser = XMLConfigParser(join(core.path, "module", "config", "plugin.xml")) self.configParser.loadData() self.config = self.configParser.getConfig() self.logger = logging.getLogger("log") diff --git a/module/file_list.py b/module/file_list.py index 84f875cad..2cdda99c4 100644 --- a/module/file_list.py +++ b/module/file_list.py @@ -23,17 +23,19 @@ LIST_VERSION = 4 -from threading import RLock -from download_thread import Status -import cPickle -import re -import module.Plugin -from operator import concat from operator import attrgetter -from os import sep +from operator import concat +from os.path import join +import re +from threading import RLock from time import sleep -from module.PullEvents import UpdateEvent, RemoveEvent, InsertEvent +import cPickle +from download_thread import Status +import module.Plugin +from module.PullEvents import InsertEvent +from module.PullEvents import RemoveEvent +from module.PullEvents import UpdateEvent class NoSuchElementException(Exception): pass @@ -57,7 +59,7 @@ class File_List(object): def load(self): self.lock.acquire() try: - pkl_file = open('module' + sep + 'links.pkl', 'rb') + pkl_file = open(join(self.core.path, 'module', 'links.pkl'), 'rb') obj = cPickle.load(pkl_file) except: obj = False @@ -104,7 +106,7 @@ class File_List(object): pdata["queue"] = [PyLoadPackageData().set(x) for x in self.data["queue"]] pdata["collector"] = [PyLoadFileData().set(x) for x in self.data["collector"]] - output = open('module' + sep + 'links.pkl', 'wb') + output = open(join(self.core.path, 'module', 'links.pkl'), 'wb') cPickle.dump(pdata, output, -1) self.lock.release() @@ -214,7 +216,7 @@ class File_List(object): """ pyfile = PyLoadFile(url, collector.file_list) pyfile.id = collector._getFreeID() - pyfile.folder = collector.file_list.download_folder + pyfile.folder = collector.file_list.download_folder collector.file_list.lock.acquire() collector.file_list.data["collector"].append(pyfile) collector.file_list.lock.release() @@ -248,7 +250,7 @@ class File_List(object): """ returns a free id """ - ids = [ pypack.data["id"] for pypack in packager.file_list.data["packages"] + packager.file_list.data["queue"]] + ids = [pypack.data["id"] for pypack in packager.file_list.data["packages"] + packager.file_list.data["queue"]] id = 1 while id in ids: diff --git a/module/plugins/hooks/ExternalScripts.py b/module/plugins/hooks/ExternalScripts.py index 30bd43770..01daab87a 100644 --- a/module/plugins/hooks/ExternalScripts.py +++ b/module/plugins/hooks/ExternalScripts.py @@ -36,17 +36,20 @@ class ExternalScripts(Hook): self.core = core self.scripts = {} - script_folders = ['scripts'+sep+'download_preparing'+sep, - 'scripts'+sep+'download_finished'+sep, - 'scripts'+sep+'package_finished'+sep, - 'scripts'+sep+'before_reconnect'+sep, - 'scripts'+sep+'after_reconnect'+sep]
+ script_folders = [join(core.path, 'scripts','download_preparing'), + join(core.path,'scripts','download_finished'), + join(core.path,'scripts','package_finished'), + join(core.path,'scripts','before_reconnect'), + join(core.path,'scripts','after_reconnect')] + + folder = core.make_path("scripts") + + self.core.check_file(folder, _("folders for scripts"), True) self.core.check_file(script_folders, _("folders for scripts"), True) f = lambda x: False if x.startswith("#") or x.endswith("~") else True self.scripts = {} - folder = join(self.core.path, "scripts") self.scripts['download_preparing'] = filter(f, listdir(join(folder, 'download_preparing'))) self.scripts['download_finished'] = filter(f, listdir(join(folder, 'download_finished'))) diff --git a/module/plugins/hooks/LinuxFileEvents.py b/module/plugins/hooks/LinuxFileEvents.py index f71521b33..ed206866a 100644 --- a/module/plugins/hooks/LinuxFileEvents.py +++ b/module/plugins/hooks/LinuxFileEvents.py @@ -35,7 +35,7 @@ class LinuxFileEvents(Hook): if not os.name == "posix": return - self.core.check_file("container", _("folder for container"), True) + self.core.check_file(self.core.make_path("container"), _("folder for container"), True) self.core.check_install("pyinotify", _("pyinotify for LinuxFileEvents")) try: diff --git a/module/web/cnl/views.py b/module/web/cnl/views.py index 9d309dbf6..30d72ca5a 100644 --- a/module/web/cnl/views.py +++ b/module/web/cnl/views.py @@ -37,7 +37,7 @@ def local_check(function): @local_check def flash(request): - return HttpResponse() + return HttpResponse("JDownloader") @local_check def add(request): @@ -73,13 +73,25 @@ def addcrypted2(request): crypted = base64.standard_b64decode(unquote(crypted.replace(" ", "+"))) - jk = re.findall(r"return ('|\")(.+)('|\")", jk)[0][1] + print jk + try: + jk = re.findall(r"return ('|\")(.+)('|\")", jk)[0][1] + except: + ## Test for some known js functions to decode + if jk.find("dec") > -1 and jk.find("org") > -1: + org = re.findall(r"var org = ('|\")([^\"']+)", jk)[0][1] + jk = list(org) + jk.reverse() + jk = "".join(jk) + print jk + + Key = binascii.unhexlify(jk) IV = Key obj = AES.new(Key, AES.MODE_CBC, IV) - result = obj.decrypt(crypted).replace("\x00", "").split("\n") + result = obj.decrypt(crypted).replace("\x00", "").replace("\r","").split("\n") result = filter(lambda x: x != "", result) @@ -103,9 +115,9 @@ def flashgot(request): @local_check def crossdomain(request): - rep = "<?xml version=\"1.0\"?>\r\n" - rep += "<!DOCTYPE cross-domain-policy SYSTEM \"http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd\">\r\n" - rep += "<cross-domain-policy>\r\n" - rep += "<allow-access-from domain=\"*\" />\r\n" - rep += "</cross-domain-policy>\r\n" + rep = "<?xml version=\"1.0\"?>\n" + rep += "<!DOCTYPE cross-domain-policy SYSTEM \"http://www.macromedia.com/xml/dtds/cross-domain-policy.dtd\">\n" + rep += "<cross-domain-policy>\n" + rep += "<allow-access-from domain=\"*\" />\n" + rep += "</cross-domain-policy>" return HttpResponse(rep)
\ No newline at end of file diff --git a/module/web/urls.py b/module/web/urls.py index b753890ee..165c7e146 100644 --- a/module/web/urls.py +++ b/module/web/urls.py @@ -16,7 +16,7 @@ urlpatterns = patterns('', (r'^admin/', include(admin.site.urls)), # django 1.0 not working (r'^json/', include('ajax.urls')), (r'^flashgot$', 'cnl.views.flashgot'), - (r'^flash(got)?/', include('cnl.urls')), + (r'^flash(got)?/?', include('cnl.urls')), (r'^crossdomain.xml$', 'cnl.views.crossdomain'), (r'^favicon\.ico$', 'django.views.generic.simple.redirect_to', {'url': '/media/img/favicon.ico'}), (r'^media/(?P<path>.*)$', 'django.views.static.serve', diff --git a/pyLoadCore.py b/pyLoadCore.py index 5ea4b43c5..01646212f 100755 --- a/pyLoadCore.py +++ b/pyLoadCore.py @@ -1,636 +1,644 @@ -#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; either version 3 of the License,
- or (at your option) any later version.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, see <http://www.gnu.org/licenses/>.
-
- @author: spoob
- @author: sebnapi
- @author: RaNaN
- @author: mkaay
- @version: v0.3.1
-"""
-
-CURRENT_VERSION = '0.3.1'
-from getopt import getopt
-import gettext
-from glob import glob
-from imp import find_module
-import logging
-import logging.handlers
-from operator import attrgetter
-from os import execv
-from os import makedirs
-from os import remove
-from os import sep
-from os.path import basename
-from os.path import dirname
-from os.path import exists
-from os.path import join
-from re import sub
-import subprocess
-import sys
-from sys import argv
-from sys import executable
-from sys import exit
-from sys import path
-from sys import stdout
-from sys import version_info
-from tempfile import NamedTemporaryFile
-import thread
-import time
-from time import sleep
-from xmlrpclib import Binary
-
-from module.CaptchaManager import CaptchaManager
-from module.HookManager import HookManager
-from module.PullEvents import PullManager
-from module.XMLConfigParser import XMLConfigParser
-from module.file_list import File_List
-from module.network.Request import Request
-import module.remote.SecureXMLRPCServer as Server
-from module.thread_list import Thread_List
-from module.web.ServerThread import WebServer
-
-class Core(object):
- """ pyLoad Core """
- def __init__(self):
- self.arg_links = []
- if len(argv) > 1:
- try:
- options, arguments = getopt(argv[1:], 'vcl:')
- for option, argument in options:
- if option == "-v":
- print "pyLoad", CURRENT_VERSION
- exit()
- elif option == "-c":
- remove(join("module", "links.pkl"))
- print "Removed Linklist"
- elif option == "-l":
- self.arg_links.append(argument)
- print "Added %s" % argument
- except:
- print 'Unknown Argument(s) "%s"' % " ".join(argv[1:])
-
- def toggle_pause(self):
- if self.thread_list.pause:
- self.thread_list.pause = False
- return False
- elif not self.thread_list.pause:
- self.thread_list.pause = True
- return True
-
- def start(self):
- """ starts the machine"""
- self.path = dirname(__file__)
-
- self.config = {}
- self.plugins_avaible = {}
-
- self.plugin_folder = join("module", "plugins")
-
- self.xmlconfig = XMLConfigParser(join(self.path, "module", "config", "core.xml"))
- self.config = self.xmlconfig.getConfig()
- self.parser_plugins = XMLConfigParser(join(self.path, "module", "config", "plugin.xml"))
- #~ self.config_plugins = self.parser_plugins.getConfig()
-
- self.do_kill = False
- self.do_restart = False
- translation = gettext.translation("pyLoad", join(self.path, "locale"), languages=[self.config['general']['language']])
- try:
- translation.ugettext("äöü")
- unicode = True
- except:
- unicode = False
-
- #@TODO cleaner method, if possible
-
- translation.install(unicode)
-
- self.check_install("Crypto", _("pycrypto to decode container files"))
- self.check_install("Image", _("Python Image Libary (PIL) for captha reading"))
- self.check_install("pycurl", _("pycurl for lower memory footprint while downloading"))
- self.check_install("django", _("Django for webinterface"))
- self.check_install("tesseract", _("tesseract for captcha reading"), False)
- self.check_install("gocr", _("gocr for captcha reading"), False)
-
- self.check_file(self.config['log']['log_folder'], _("folder for logs"), True)
- self.check_file(self.config['general']['download_folder'], _("folder for downloads"), True)
- self.check_file(self.config['general']['link_file'], _("file for links"))
- self.check_file(self.config['general']['failed_file'], _("file for failed links"))
-
- if self.config['ssl']['activated']:
- self.check_install("OpenSSL", _("OpenSSL for secure connection"), True)
- self.check_file(self.config['ssl']['cert'], _("ssl certificate"), False, True)
- self.check_file(self.config['ssl']['key'], _("ssl key"), False, True)
-
- self.downloadSpeedLimit = int(self.xmlconfig.get("general", "download_speed_limit", 0))
-
- if self.config['general']['debug_mode']:
- self.init_logger(logging.DEBUG) # logging level
- else:
- self.init_logger(logging.INFO) # logging level
-
- self.init_hooks()
- path.append(self.plugin_folder)
- self.create_plugin_index()
-
- self.lastGuiConnected = 0
-
- self.server_methods = ServerMethods(self)
- self.file_list = File_List(self)
- self.pullManager = PullManager(self)
- self.thread_list = Thread_List(self)
- self.captchaManager = CaptchaManager(self)
-
- self.last_update_check = 0
- self.update_check_interval = 1800
- self.update_available = self.check_update()
- self.logger.info(_("Downloadtime: %s") % self.server_methods.is_time_download())
-
- self.init_server()
- self.init_webserver()
-
- linkFile = self.config['general']['link_file']
- packs = self.server_methods.get_queue()
- found = False
- for data in packs:
- if data["package_name"] == linkFile:
- found = data["id"]
- break
- if found == False:
- pid = self.file_list.packager.addNewPackage(package_name=linkFile)
- else:
- pid = found
- lid = self.file_list.collector.addLink(linkFile)
- self.file_list.packager.addFileToPackage(pid, self.file_list.collector.popFile(lid))
- if self.arg_links:
- for link in self.arg_links:
- lid = self.file_list.collector.addLink(link)
- self.file_list.packager.addFileToPackage(pid, self.file_list.collector.popFile(lid))
-
- self.file_list.packager.pushPackage2Queue(pid)
- self.file_list.continueAborted()
-
- while True:
- sleep(2)
- if self.do_restart:
- self.logger.info(_("restarting pyLoad"))
- self.restart()
- if self.do_kill:
- self.shutdown()
- self.logger.info(_("pyLoad quits"))
- exit()
- if self.last_update_check + self.update_check_interval <= time.time():
- self.update_available = self.check_update()
-
- def init_server(self):
- try:
- server_addr = (self.config['remote']['listenaddr'], int(self.config['remote']['port']))
- usermap = {self.config['remote']['username']: self.config['remote']['password']}
- if self.config['ssl']['activated']:
- self.server = Server.SecureXMLRPCServer(server_addr, self.config['ssl']['cert'], self.config['ssl']['key'], usermap)
- self.logger.info(_("Secure XMLRPC Server Started"))
- else:
- self.server = Server.AuthXMLRPCServer(server_addr, usermap)
- self.logger.info(_("Auth XMLRPC Server Started"))
-
- self.server.register_instance(self.server_methods)
-
- thread.start_new_thread(self.server.serve_forever, ())
- except Exception, e:
- self.logger.error(_("Failed starting XMLRPC server CLI and GUI will not be available: %s") % str(e))
- if self.config['general']['debug_mode']:
- import traceback
- traceback.print_exc()
-
-
- def init_webserver(self):
- #pyloadDBFile = join(self.path, "module", "web", "pyload.db")
- #pyloadDefaultDBFile = join(self.path, "module", "web", "pyload_default.db")
- #if not exists(pyloadDBFile):
- # copyfile(pyloadDefaultDBFile, pyloadDBFile)
- if self.config['webinterface']['activated']:
- self.webserver = WebServer(self)
- self.webserver.start()
-
- def init_logger(self, level):
- console = logging.StreamHandler(stdout)
- frm = logging.Formatter("%(asctime)s: %(levelname)-8s %(message)s", "%d.%m.%Y %H:%M:%S")
- console.setFormatter(frm)
- self.logger = logging.getLogger("log") # settable in config
-
- if self.config['log']['file_log']:
- file_handler = logging.handlers.RotatingFileHandler(self.config['log']['log_folder'] + sep + 'log.txt', maxBytes=102400, backupCount=int(self.config['log']['log_count'])) #100 kib each
- file_handler.setFormatter(frm)
- self.logger.addHandler(file_handler)
-
- self.logger.addHandler(console) #if console logging
- self.logger.setLevel(level)
-
- def init_hooks(self):
- self.hookManager = HookManager(self)
-
- def check_install(self, check_name, legend, python=True, essential=False):
- """check wether needed tools are installed"""
- try:
- if python:
- find_module(check_name)
- else:
- pipe = subprocess.PIPE
- subprocess.Popen(check_name, stdout=pipe, stderr=pipe)
- except:
- print _("Install %s") % legend
- if essential: exit()
-
- def check_file(self, check_names, description="", folder=False, empty=True, essential=False):
- """check wether needed files are exists"""
- tmp_names = []
- if not type(check_names) == list:
- tmp_names.append(check_names)
- else:
- tmp_names.extend(check_names)
- file_created = True
- file_exists = True
- for tmp_name in tmp_names:
- if not exists(tmp_name):
- file_exists = False
- if empty:
- try:
- if folder:
- tmp_name = tmp_name.replace("/", sep)
- makedirs(tmp_name)
- else:
- open(tmp_name, "w")
- except:
- file_created = False
- else:
- file_created = False
- if not file_exists:
- if file_created:
- print _("%s created") % description
- else:
- if not empty:
- print _("could not find %s: %s") % (description, tmp_name)
- else:
- print _("could not create %s: %s") % (description, tmp_name)
- if essential:
- exit()
-
- def isGUIConnected(self):
- return self.lastGuiConnected + 10 > time.time()
-
- def restart(self):
- self.shutdown()
- execv(executable, [executable, "pyLoadCore.py"])
-
- def create_plugin_index(self):
- plugins = glob(join(self.plugin_folder, "hoster", "*.py"))
- plugins += glob(join(self.plugin_folder, "decrypter", "*.py"))
- plugins += glob(join(self.plugin_folder, "container", "*.py"))
- plugins += glob(join(self.plugin_folder, "container", "DLC_*.pyc"))
- for file_handler in plugins:
- plugin_pattern = ""
- plugin_file = sub("(\.pyc|\.py)", "", basename(file_handler))
- if plugin_file.startswith("DLC"):
- if plugin_file == "DLC_25" and not version_info < (2, 6):
- continue
- if plugin_file == "DLC_26" and not version_info > (2, 6):
- continue
- plugin_pattern = "(?!http://).*\.dlc"
- else:
- for line in open(file_handler, "r").readlines():
- if "props['pattern']" in line:
- plugin_pattern = line.split("r\"")[1].split("\"")[0]
- break
- if plugin_pattern != "":
- self.plugins_avaible[plugin_file] = plugin_pattern
- self.logger.debug(_("%s added") % plugin_file)
- self.logger.info(_("created index of plugins"))
-
- def compare_time(self, start, end):
-
- start = map(int, start)
- end = map(int, end)
-
- if start == end: return True
-
- now = list(time.localtime()[3:5])
- if start < now and end > now: return True
- elif start > end and (now > start or now < end): return True
- elif start < now and end < now and start > end: return True
- else: return False
-
- def getMaxSpeed(self):
- return self.downloadSpeedLimit
-
- def shutdown(self):
- self.logger.info(_("shutting down..."))
- if self.config['webinterface']['activated']:
- self.webserver.quit()
- self.webserver.join()
- for thread in self.thread_list.threads:
- thread.shutdown = True
- self.thread_list.stopAllDownloads()
- for thread in self.thread_list.threads:
- thread.join(15)
- self.file_list.save()
-
- def check_update(self):
- try:
- if self.config['updates']['search_updates']:
- version_check = Request().load("http://get.pyload.org/check/%s/" % (CURRENT_VERSION, ))
- if version_check == "":
- self.logger.info(_("No Updates for pyLoad"))
- return False
- else:
- self.logger.info(_("New pyLoad Version %s available") % version_check)
- return True
- else:
- return False
- finally:
- self.last_update_check = time.time()
-
- def install_update(self):
- if self.config['updates']['search_updates']:
- if self.core.config['updates']['install_updates']:
- version_check = Request().load("http://get.pyload.org/get/update/%s/" % (CURRENT_VERSION, ))
- else:
- version_check = Request().load("http://get.pyload.org/check/%s/" % (CURRENT_VERSION, ))
- if version_check == "":
- return False
- else:
- if self.config['updates']['install_updates']:
- try:
- tmp_zip_name = __import__("tempfile").NamedTemporaryFile(suffix=".zip").name
- tmp_zip = open(tmp_zip_name, 'wb')
- tmp_zip.write(version_check)
- tmp_zip.close()
- __import__("module.Unzip", globals(), locals(), "Unzip", -1).Unzip().extract(tmp_zip_name, "Test/")
- return True
- except:
- self.logger.info(_("Auto install Failed"))
- return False
- else:
- return False
- else:
- return False
-
- ####################################
- ########## XMLRPC Methods ##########
- ####################################
-
-class ServerMethods():
- def __init__(self, core):
- self.core = core
-
- def status_downloads(self):
- downloads = []
- for pyfile in self.core.thread_list.py_downloading:
- download = {}
- download['id'] = pyfile.id
- download['name'] = pyfile.status.filename
- download['speed'] = pyfile.status.get_speed()
- download['eta'] = pyfile.status.get_ETA()
- download['kbleft'] = pyfile.status.kB_left()
- download['size'] = pyfile.status.size()
- download['percent'] = pyfile.status.percent()
- download['status'] = pyfile.status.type
- download['wait_until'] = pyfile.status.waituntil
- download['package'] = pyfile.package.data["package_name"]
- downloads.append(download)
- return downloads
-
- def get_conf_val(self, cat, var):
- if var != "username" and var != "password":
- return self.core.config[cat][var]
- else:
- raise Exception("not allowed!")
-
- def pause_server(self):
- self.core.thread_list.pause = True
-
- def unpause_server(self):
- self.core.thread_list.pause = False
-
- def toggle_pause(self):
- if self.core.thread_list.pause:
- self.core.thread_list.pause = False
- else:
- self.core.thread_list.pause = True
- return self.core.thread_list.pause
-
- def status_server(self):
- status = {}
- status['pause'] = self.core.thread_list.pause
- status['activ'] = len(self.core.thread_list.py_downloading)
- status['queue'] = self.core.file_list.countDownloads()
- status['total'] = len(self.core.file_list.data['queue'])
- status['speed'] = 0
-
- for pyfile in self.core.thread_list.py_downloading:
- status['speed'] += pyfile.status.get_speed()
-
- return status
-
- def file_exists(self, path): #@XXX: security?!
- return exists(path)
-
- def get_server_version(self):
- return CURRENT_VERSION
-
- def add_urls(self, links):
- for link in links:
- link = link.strip()
- if link.startswith("http") or exists(link):
- self.core.file_list.collector.addLink(link)
- self.core.file_list.save()
-
- def add_package(self, name, links, queue=True):
- pid = self.new_package(name)
-
- fids = map(self.core.file_list.collector.addLink, links)
- map(lambda fid: self.move_file_2_package(fid, pid), fids)
-
- if queue:
- self.core.file_list.packager.pushPackage2Queue(pid)
-
- self.core.file_list.save()
-
- def new_package(self, name):
- id = self.core.file_list.packager.addNewPackage(name)
- self.core.file_list.save()
- return id
-
- def get_package_data(self, id):
- return self.core.file_list.packager.getPackageData(id)
-
- def get_package_files(self, id):
- return self.core.file_list.packager.getPackageFiles(id)
-
- def get_file_info(self, id):
- return self.core.file_list.getFileInfo(id)
-
- def del_links(self, ids):
- for id in ids:
- try:
- self.core.file_list.collector.removeFile(id)
- except:
- self.core.file_list.packager.removeFile(id)
- self.core.file_list.save()
-
- def del_packages(self, ids):
- map(self.core.file_list.packager.removePackage, ids)
- self.core.file_list.save()
-
- def kill(self):
- self.core.do_kill = True
- return True
-
- def restart(self):
- self.core.do_restart = True
-
- def get_queue(self):
- return map(attrgetter("data"), self.core.file_list.data["queue"])
-
- def get_collector_packages(self):
- return map(attrgetter("data"), self.core.file_list.data["packages"])
-
- def get_collector_files(self):
- return map(attrgetter("id"), self.core.file_list.data["collector"])
-
- def move_file_2_package(self, fid, pid):
- try:
- pyfile = self.core.file_list.collector.getFile(fid)
- self.core.file_list.packager.addFileToPackage(pid, pyfile)
- except:
- return
- else:
- self.core.file_list.collector.removeFile(fid)
-
- def push_package_2_queue(self, id):
- self.core.file_list.packager.pushPackage2Queue(id)
-
- def restart_package(self, packid):
- map(self.core.file_list.packager.resetFileStatus, self.core.file_list.packager.getPackageFiles(packid))
-
- def restart_file(self, fileid):
- self.core.file_list.packager.resetFileStatus(fileid)
-
- def upload_container(self, filename, type, content):
- th = NamedTemporaryFile(mode="w", suffix="." + type, delete=False)
- th.write(content)
- path = th.name
- th.close()
- pid = self.core.file_list.packager.addNewPackage(filename)
- cid = self.core.file_list.collector.addLink(path)
- self.move_file_2_package(cid, pid)
- self.core.file_list.save()
-
- def get_log(self, offset=0):
- filename = self.core.config['log']['log_folder'] + sep + 'log.txt'
- fh = open(filename, "r")
- content = fh.read()
- fh.close()
- lines = content.splitlines()
- if offset >= len(lines):
- return None
- return lines[offset:]
-
- def stop_downloads(self):
- self.core.thread_list.stopAllDownloads()
-
- def stop_download(self, type, id):
- if type == "pack":
- ids = self.core.file_list.getPackageFiles(id)
- for fid in ids:
- self.core.file_list.packager.abortFile(fid)
- else:
- self.core.file_list.packager.abortFile(id)
-
- def update_available(self):
- return self.core.update_available
-
- def set_package_name(self, pid, name):
- self.core.file_list.packager.setPackageData(pid, package_name=name)
-
- def pull_out_package(self, pid):
- self.core.file_list.packager.pullOutPackage(pid)
-
- def is_captcha_waiting(self):
- self.core.lastGuiConnected = time.time()
- task = self.core.captchaManager.getTask()
- return not task == None
-
- def get_captcha_task(self):
- task = self.core.captchaManager.getTask()
- if task:
- task.setWatingForUser()
- c = task.getCaptcha()
- return str(task.getID()), Binary(c[0]), str(c[1])
- else:
- return None, None, None
-
- def set_captcha_result(self, tid, result):
- task = self.core.captchaManager.getTaskFromID(tid)
- if task:
- task.setResult(result)
- task.setDone()
- return True
- else:
- return False
-
- def get_events(self, uuid):
- return self.core.pullManager.getEvents(uuid)
-
- def get_full_queue(self):
- data = []
- for pack in self.core.file_list.data["queue"]:
- p = {"data":pack.data, "children":[]}
- for child in pack.files:
- info = self.core.file_list.getFileInfo(child.id)
- info["downloading"] = None
- p["children"].append(info)
- data.append(p)
- return data
-
- #def move_urls_up(self, ids):
- # for id in ids:
- # self.core.file_list.move(id)
- # self.core.file_list.save()
-
- #def move_urls_down(self, ids):
- # for id in ids:
- # self.core.file_list.move(id, 1)
- # self.core.file_list.save()
-
- def is_time_download(self):
- start = self.core.config['downloadTime']['start'].split(":")
- end = self.core.config['downloadTime']['end'].split(":")
- return self.core.compare_time(start, end)
-
- def is_time_reconnect(self):
- start = self.core.config['reconnect']['startTime'].split(":")
- end = self.core.config['reconnect']['endTime'].split(":")
- return self.core.compare_time(start, end)
-
-# And so it begins...
-if __name__ == "__main__":
- pyload_core = Core()
- try:
- pyload_core.start()
- except KeyboardInterrupt:
- pyload_core.shutdown()
- pyload_core.logger.info(_("killed pyLoad from Terminal"))
- exit()
+#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 3 of the License, + or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, see <http://www.gnu.org/licenses/>. + + @author: spoob + @author: sebnapi + @author: RaNaN + @author: mkaay + @version: v0.3.1 +""" +CURRENT_VERSION = '0.3.1' + +from getopt import getopt +import gettext +from glob import glob +from imp import find_module +import logging +import logging.handlers +from operator import attrgetter +from os import chdir +from os import execv +from os import makedirs +from os import remove +from os import sep +from os.path import abspath +from os.path import basename +from os.path import dirname +from os.path import exists +from os.path import isabs +from os.path import join +from re import sub +import subprocess +from sys import argv +from sys import executable +from sys import exit +from sys import path +from sys import stdout +from sys import version_info +from tempfile import NamedTemporaryFile +import thread +import time +from time import sleep +from xmlrpclib import Binary + +from module.CaptchaManager import CaptchaManager +from module.HookManager import HookManager +from module.PullEvents import PullManager +from module.XMLConfigParser import XMLConfigParser +from module.file_list import File_List +from module.network.Request import Request +import module.remote.SecureXMLRPCServer as Server +from module.thread_list import Thread_List +from module.web.ServerThread import WebServer + +class Core(object): + """ pyLoad Core """ + def __init__(self): + self.arg_links = [] + if len(argv) > 1: + try: + options, arguments = getopt(argv[1:], 'vcl:') + for option, argument in options: + if option == "-v": + print "pyLoad", CURRENT_VERSION + exit() + elif option == "-c": + remove(join("module", "links.pkl")) + print "Removed Linklist" + elif option == "-l": + self.arg_links.append(argument) + print "Added %s" % argument + except: + print 'Unknown Argument(s) "%s"' % " ".join(argv[1:]) + + def toggle_pause(self): + if self.thread_list.pause: + self.thread_list.pause = False + return False + elif not self.thread_list.pause: + self.thread_list.pause = True + return True + + def start(self): + """ starts the machine""" + self.path = abspath(dirname(__file__)) + chdir(self.path) + + self.config = {} + self.plugins_avaible = {} + + self.plugin_folder = self.make_path("module", "plugins") + + self.xmlconfig = XMLConfigParser(self.make_path("module", "config", "core.xml")) + self.config = self.xmlconfig.getConfig() + self.parser_plugins = XMLConfigParser(self.make_path("module", "config", "plugin.xml")) + #~ self.config_plugins = self.parser_plugins.getConfig() + + self.do_kill = False + self.do_restart = False + translation = gettext.translation("pyLoad", self.make_path("locale"), languages=[self.config['general']['language']]) + try: + translation.ugettext("äöü") + unicode = True + except: + unicode = False + + #@TODO cleaner method, if possible + + translation.install(unicode) + + self.check_install("Crypto", _("pycrypto to decode container files")) + self.check_install("Image", _("Python Image Libary (PIL) for captha reading")) + self.check_install("pycurl", _("pycurl for lower memory footprint while downloading")) + self.check_install("django", _("Django for webinterface")) + self.check_install("tesseract", _("tesseract for captcha reading"), False) + self.check_install("gocr", _("gocr for captcha reading"), False) + + self.check_file(self.make_path(self.config['log']['log_folder']), _("folder for logs"), True) + self.check_file(self.make_path(self.config['general']['download_folder']), _("folder for downloads"), True) + self.check_file(self.make_path(self.config['general']['link_file']), _("file for links")) + self.check_file(self.make_path(self.config['general']['failed_file']), _("file for failed links")) + + if self.config['ssl']['activated']: + self.check_install("OpenSSL", _("OpenSSL for secure connection"), True) + self.check_file(self.make_path(self.config['ssl']['cert']), _("ssl certificate"), False, True) + self.check_file(self.make_path(self.config['ssl']['key']), _("ssl key"), False, True) + + self.downloadSpeedLimit = int(self.xmlconfig.get("general", "download_speed_limit", 0)) + + if self.config['general']['debug_mode']: + self.init_logger(logging.DEBUG) # logging level + else: + self.init_logger(logging.INFO) # logging level + + self.init_hooks() + path.append(self.plugin_folder) + self.create_plugin_index() + + self.lastGuiConnected = 0 + + self.server_methods = ServerMethods(self) + self.file_list = File_List(self) + self.pullManager = PullManager(self) + self.thread_list = Thread_List(self) + self.captchaManager = CaptchaManager(self) + + self.last_update_check = 0 + self.update_check_interval = 1800 + self.update_available = self.check_update() + self.logger.info(_("Downloadtime: %s") % self.server_methods.is_time_download()) + + self.init_server() + self.init_webserver() + + linkFile = self.config['general']['link_file'] + + packs = self.server_methods.get_queue() + found = False + for data in packs: + if data["package_name"] == linkFile: + found = data["id"] + break + if found == False: + pid = self.file_list.packager.addNewPackage(package_name=linkFile) + else: + pid = found + lid = self.file_list.collector.addLink(linkFile) + try: + self.file_list.packager.addFileToPackage(pid, self.file_list.collector.popFile(lid)) + if self.arg_links: + for link in self.arg_links: + lid = self.file_list.collector.addLink(link) + self.file_list.packager.addFileToPackage(pid, self.file_list.collector.popFile(lid)) + + self.file_list.packager.pushPackage2Queue(pid) + self.file_list.continueAborted() + except: + pass + + while True: + sleep(2) + if self.do_restart: + self.logger.info(_("restarting pyLoad")) + self.restart() + if self.do_kill: + self.shutdown() + self.logger.info(_("pyLoad quits")) + exit() + if self.last_update_check + self.update_check_interval <= time.time(): + self.update_available = self.check_update() + + def init_server(self): + try: + server_addr = (self.config['remote']['listenaddr'], int(self.config['remote']['port'])) + usermap = {self.config['remote']['username']: self.config['remote']['password']} + if self.config['ssl']['activated']: + self.server = Server.SecureXMLRPCServer(server_addr, self.config['ssl']['cert'], self.config['ssl']['key'], usermap) + self.logger.info(_("Secure XMLRPC Server Started")) + else: + self.server = Server.AuthXMLRPCServer(server_addr, usermap) + self.logger.info(_("Auth XMLRPC Server Started")) + + self.server.register_instance(self.server_methods) + + thread.start_new_thread(self.server.serve_forever, ()) + except Exception, e: + self.logger.error(_("Failed starting XMLRPC server CLI and GUI will not be available: %s") % str(e)) + if self.config['general']['debug_mode']: + import traceback + traceback.print_exc() + + + def init_webserver(self): + if self.config['webinterface']['activated']: + self.webserver = WebServer(self) + self.webserver.start() + + def init_logger(self, level): + console = logging.StreamHandler(stdout) + frm = logging.Formatter("%(asctime)s: %(levelname)-8s %(message)s", "%d.%m.%Y %H:%M:%S") + console.setFormatter(frm) + self.logger = logging.getLogger("log") # settable in config + + if self.config['log']['file_log']: + file_handler = logging.handlers.RotatingFileHandler(join(self.path, self.config['log']['log_folder'], 'log.txt'), maxBytes=102400, backupCount=int(self.config['log']['log_count'])) #100 kib each + file_handler.setFormatter(frm) + self.logger.addHandler(file_handler) + + self.logger.addHandler(console) #if console logging + self.logger.setLevel(level) + + def init_hooks(self): + self.hookManager = HookManager(self) + + def check_install(self, check_name, legend, python=True, essential=False): + """check wether needed tools are installed""" + try: + if python: + find_module(check_name) + else: + pipe = subprocess.PIPE + subprocess.Popen(check_name, stdout=pipe, stderr=pipe) + except: + print _("Install %s") % legend + if essential: exit() + + def check_file(self, check_names, description="", folder=False, empty=True, essential=False): + """check wether needed files exists""" + tmp_names = [] + if not type(check_names) == list: + tmp_names.append(check_names) + else: + tmp_names.extend(check_names) + file_created = True + file_exists = True + for tmp_name in tmp_names: + if not exists(tmp_name): + file_exists = False + if empty: + try: + if folder: + tmp_name = tmp_name.replace("/", sep) + makedirs(tmp_name) + else: + open(tmp_name, "w") + except: + file_created = False + else: + file_created = False + if not file_exists: + if file_created: + print _("%s created") % description + else: + if not empty: + print _("could not find %s: %s") % (description, tmp_name) + else: + print _("could not create %s: %s") % (description, tmp_name) + if essential: + exit() + + def isGUIConnected(self): + return self.lastGuiConnected + 10 > time.time() + + def restart(self): + self.shutdown() + execv(executable, [executable, "pyLoadCore.py"]) + + def create_plugin_index(self): + plugins = glob(join(self.plugin_folder, "hoster", "*.py")) + plugins += glob(join(self.plugin_folder, "decrypter", "*.py")) + plugins += glob(join(self.plugin_folder, "container", "*.py")) + plugins += glob(join(self.plugin_folder, "container", "DLC_*.pyc")) + for file_handler in plugins: + plugin_pattern = "" + plugin_file = sub("(\.pyc|\.py)", "", basename(file_handler)) + if plugin_file.startswith("DLC"): + if plugin_file == "DLC_25" and not version_info < (2, 6): + continue + if plugin_file == "DLC_26" and not version_info > (2, 6): + continue + plugin_pattern = "(?!http://).*\.dlc" + else: + for line in open(file_handler, "r").readlines(): + if "props['pattern']" in line: + plugin_pattern = line.split("r\"")[1].split("\"")[0] + break + if plugin_pattern != "": + self.plugins_avaible[plugin_file] = plugin_pattern + self.logger.debug(_("%s added") % plugin_file) + self.logger.info(_("created index of plugins")) + + def compare_time(self, start, end): + + start = map(int, start) + end = map(int, end) + + if start == end: return True + + now = list(time.localtime()[3:5]) + if start < now and end > now: return True + elif start > end and (now > start or now < end): return True + elif start < now and end < now and start > end: return True + else: return False + + def getMaxSpeed(self): + return self.downloadSpeedLimit + + def shutdown(self): + self.logger.info(_("shutting down...")) + if self.config['webinterface']['activated']: + self.webserver.quit() + self.webserver.join() + for thread in self.thread_list.threads: + thread.shutdown = True + self.thread_list.stopAllDownloads() + for thread in self.thread_list.threads: + thread.join(15) + self.file_list.save() + + def check_update(self): + try: + if self.config['updates']['search_updates']: + version_check = Request().load("http://get.pyload.org/check/%s/" % (CURRENT_VERSION, )) + if version_check == "": + self.logger.info(_("No Updates for pyLoad")) + return False + else: + self.logger.info(_("New pyLoad Version %s available") % version_check) + return True + else: + return False + finally: + self.last_update_check = time.time() + + def install_update(self): + if self.config['updates']['search_updates']: + if self.core.config['updates']['install_updates']: + version_check = Request().load("http://get.pyload.org/get/update/%s/" % (CURRENT_VERSION, )) + else: + version_check = Request().load("http://get.pyload.org/check/%s/" % (CURRENT_VERSION, )) + if version_check == "": + return False + else: + if self.config['updates']['install_updates']: + try: + tmp_zip_name = __import__("tempfile").NamedTemporaryFile(suffix=".zip").name + tmp_zip = open(tmp_zip_name, 'wb') + tmp_zip.write(version_check) + tmp_zip.close() + __import__("module.Unzip", globals(), locals(), "Unzip", -1).Unzip().extract(tmp_zip_name, "Test/") + return True + except: + self.logger.info(_("Auto install Failed")) + return False + else: + return False + else: + return False + + def make_path(self, * args): + if isabs(args[0]): + return args[0] + else: + return join(self.path, * args) + + #################################### + ########## XMLRPC Methods ########## + #################################### + +class ServerMethods(): + def __init__(self, core): + self.core = core + + def status_downloads(self): + downloads = [] + for pyfile in self.core.thread_list.py_downloading: + download = {} + download['id'] = pyfile.id + download['name'] = pyfile.status.filename + download['speed'] = pyfile.status.get_speed() + download['eta'] = pyfile.status.get_ETA() + download['kbleft'] = pyfile.status.kB_left() + download['size'] = pyfile.status.size() + download['percent'] = pyfile.status.percent() + download['status'] = pyfile.status.type + download['wait_until'] = pyfile.status.waituntil + download['package'] = pyfile.package.data["package_name"] + downloads.append(download) + return downloads + + def get_conf_val(self, cat, var): + if var != "username" and var != "password": + return self.core.config[cat][var] + else: + raise Exception("not allowed!") + + def pause_server(self): + self.core.thread_list.pause = True + + def unpause_server(self): + self.core.thread_list.pause = False + + def toggle_pause(self): + if self.core.thread_list.pause: + self.core.thread_list.pause = False + else: + self.core.thread_list.pause = True + return self.core.thread_list.pause + + def status_server(self): + status = {} + status['pause'] = self.core.thread_list.pause + status['activ'] = len(self.core.thread_list.py_downloading) + status['queue'] = self.core.file_list.countDownloads() + status['total'] = len(self.core.file_list.data['queue']) + status['speed'] = 0 + + for pyfile in self.core.thread_list.py_downloading: + status['speed'] += pyfile.status.get_speed() + + return status + + def file_exists(self, path): #@XXX: security?! + return exists(path) + + def get_server_version(self): + return CURRENT_VERSION + + def add_urls(self, links): + for link in links: + link = link.strip() + if link.startswith("http") or exists(link): + self.core.file_list.collector.addLink(link) + self.core.file_list.save() + + def add_package(self, name, links, queue=True): + pid = self.new_package(name) + + fids = map(self.core.file_list.collector.addLink, links) + map(lambda fid: self.move_file_2_package(fid, pid), fids) + + if queue: + self.core.file_list.packager.pushPackage2Queue(pid) + + self.core.file_list.save() + + def new_package(self, name): + id = self.core.file_list.packager.addNewPackage(name) + self.core.file_list.save() + return id + + def get_package_data(self, id): + return self.core.file_list.packager.getPackageData(id) + + def get_package_files(self, id): + return self.core.file_list.packager.getPackageFiles(id) + + def get_file_info(self, id): + return self.core.file_list.getFileInfo(id) + + def del_links(self, ids): + for id in ids: + try: + self.core.file_list.collector.removeFile(id) + except: + self.core.file_list.packager.removeFile(id) + self.core.file_list.save() + + def del_packages(self, ids): + map(self.core.file_list.packager.removePackage, ids) + self.core.file_list.save() + + def kill(self): + self.core.do_kill = True + return True + + def restart(self): + self.core.do_restart = True + + def get_queue(self): + return map(attrgetter("data"), self.core.file_list.data["queue"]) + + def get_collector_packages(self): + return map(attrgetter("data"), self.core.file_list.data["packages"]) + + def get_collector_files(self): + return map(attrgetter("id"), self.core.file_list.data["collector"]) + + def move_file_2_package(self, fid, pid): + try: + pyfile = self.core.file_list.collector.getFile(fid) + self.core.file_list.packager.addFileToPackage(pid, pyfile) + except: + return + else: + self.core.file_list.collector.removeFile(fid) + + def push_package_2_queue(self, id): + self.core.file_list.packager.pushPackage2Queue(id) + + def restart_package(self, packid): + map(self.core.file_list.packager.resetFileStatus, self.core.file_list.packager.getPackageFiles(packid)) + + def restart_file(self, fileid): + self.core.file_list.packager.resetFileStatus(fileid) + + def upload_container(self, filename, type, content): + th = NamedTemporaryFile(mode="w", suffix="." + type, delete=False) + th.write(content) + path = th.name + th.close() + pid = self.core.file_list.packager.addNewPackage(filename) + cid = self.core.file_list.collector.addLink(path) + self.move_file_2_package(cid, pid) + self.core.file_list.save() + + def get_log(self, offset=0): + filename = self.core.config['log']['log_folder'] + sep + 'log.txt' + fh = open(filename, "r") + content = fh.read() + fh.close() + lines = content.splitlines() + if offset >= len(lines): + return None + return lines[offset:] + + def stop_downloads(self): + self.core.thread_list.stopAllDownloads() + + def stop_download(self, type, id): + if type == "pack": + ids = self.core.file_list.getPackageFiles(id) + for fid in ids: + self.core.file_list.packager.abortFile(fid) + else: + self.core.file_list.packager.abortFile(id) + + def update_available(self): + return self.core.update_available + + def set_package_name(self, pid, name): + self.core.file_list.packager.setPackageData(pid, package_name=name) + + def pull_out_package(self, pid): + self.core.file_list.packager.pullOutPackage(pid) + + def is_captcha_waiting(self): + self.core.lastGuiConnected = time.time() + task = self.core.captchaManager.getTask() + return not task == None + + def get_captcha_task(self): + task = self.core.captchaManager.getTask() + if task: + task.setWatingForUser() + c = task.getCaptcha() + return str(task.getID()), Binary(c[0]), str(c[1]) + else: + return None, None, None + + def set_captcha_result(self, tid, result): + task = self.core.captchaManager.getTaskFromID(tid) + if task: + task.setResult(result) + task.setDone() + return True + else: + return False + + def get_events(self, uuid): + return self.core.pullManager.getEvents(uuid) + + def get_full_queue(self): + data = [] + for pack in self.core.file_list.data["queue"]: + p = {"data":pack.data, "children":[]} + for child in pack.files: + info = self.core.file_list.getFileInfo(child.id) + info["downloading"] = None + p["children"].append(info) + data.append(p) + return data + + #def move_urls_up(self, ids): + # for id in ids: + # self.core.file_list.move(id) + # self.core.file_list.save() + + #def move_urls_down(self, ids): + # for id in ids: + # self.core.file_list.move(id, 1) + # self.core.file_list.save() + + def is_time_download(self): + start = self.core.config['downloadTime']['start'].split(":") + end = self.core.config['downloadTime']['end'].split(":") + return self.core.compare_time(start, end) + + def is_time_reconnect(self): + start = self.core.config['reconnect']['startTime'].split(":") + end = self.core.config['reconnect']['endTime'].split(":") + return self.core.compare_time(start, end) + +# And so it begins... +if __name__ == "__main__": + pyload_core = Core() + try: + pyload_core.start() + except KeyboardInterrupt: + pyload_core.shutdown() + pyload_core.logger.info(_("killed pyLoad from Terminal")) + exit()
\ No newline at end of file diff --git a/systemCheck.py b/systemCheck.py index 79d8600a2..831f76274 100644 --- a/systemCheck.py +++ b/systemCheck.py @@ -68,7 +68,13 @@ def main(): import pycurl except: core_err.append("Please install py-curl to use pyLoad.") -#@TODO version > 7.19 + + + try: + from pycurl import AUTOREFERER + except: + core_err.append("Your py-curl version is to old, please upgrade!") + try: import Image except: |