From 8606b72470e0e506d9da26f190a2880d47d289a4 Mon Sep 17 00:00:00 2001 From: Walter Purcaro Date: Sun, 10 May 2015 17:01:05 +0200 Subject: Improve and fix file structure a bit --- pyload/Core.py | 12 +- pyload/Thread/Addon.py | 70 +++++ pyload/Thread/Decrypter.py | 105 +++++++ pyload/Thread/Download.py | 213 ++++++++++++++ pyload/Thread/Info.py | 221 ++++++++++++++ pyload/Thread/Plugin.py | 132 +++++++++ pyload/Thread/Server.py | 122 ++++++++ pyload/Thread/__init__.py | 7 + pyload/__init__.py | 24 +- pyload/api/__init__.py | 2 +- pyload/cli/AddPackage.py | 2 +- pyload/cli/Cli.py | 578 ------------------------------------ pyload/cli/ManageFiles.py | 2 +- pyload/cli/__init__.py | 583 ++++++++++++++++++++++++++++++++++++- pyload/config/Setup.py | 4 +- pyload/database/File.py | 7 +- pyload/database/Storage.py | 5 +- pyload/database/User.py | 2 +- pyload/database/__init__.py | 8 +- pyload/datatype/__init__.py | 3 + pyload/manager/Addon.py | 2 +- pyload/manager/Scheduler.py | 141 +++++++++ pyload/manager/Thread.py | 6 +- pyload/manager/event/Scheduler.py | 141 --------- pyload/manager/event/__init__.py | 1 - pyload/manager/thread/Addon.py | 70 ----- pyload/manager/thread/Decrypter.py | 105 ------- pyload/manager/thread/Download.py | 213 -------------- pyload/manager/thread/Info.py | 221 -------------- pyload/manager/thread/Plugin.py | 132 --------- pyload/manager/thread/Server.py | 122 -------- pyload/manager/thread/__init__.py | 1 - pyload/plugin/Extractor.py | 2 +- pyload/webui/__init__.py | 2 +- 34 files changed, 1633 insertions(+), 1628 deletions(-) create mode 100644 pyload/Thread/Addon.py create mode 100644 pyload/Thread/Decrypter.py create mode 100644 pyload/Thread/Download.py create mode 100644 pyload/Thread/Info.py create mode 100644 pyload/Thread/Plugin.py create mode 100644 pyload/Thread/Server.py create mode 100644 pyload/Thread/__init__.py delete mode 100644 pyload/cli/Cli.py create mode 100644 pyload/manager/Scheduler.py delete mode 100644 pyload/manager/event/Scheduler.py delete mode 100644 pyload/manager/event/__init__.py delete mode 100644 pyload/manager/thread/Addon.py delete mode 100644 pyload/manager/thread/Decrypter.py delete mode 100644 pyload/manager/thread/Download.py delete mode 100644 pyload/manager/thread/Info.py delete mode 100644 pyload/manager/thread/Plugin.py delete mode 100644 pyload/manager/thread/Server.py delete mode 100644 pyload/manager/thread/__init__.py (limited to 'pyload') diff --git a/pyload/Core.py b/pyload/Core.py index 2230b3fed..2b356b9fb 100755 --- a/pyload/Core.py +++ b/pyload/Core.py @@ -25,15 +25,15 @@ from sys import argv, executable, exit from time import time, sleep from pyload import remote -from pyload.database import DatabaseBackend, FileHandler +from pyload.Database import DatabaseBackend, FileHandler from pyload.config.Parser import ConfigParser from pyload.manager.Account import AccountManager from pyload.manager.Captcha import CaptchaManager from pyload.manager.Event import PullManager from pyload.manager.Plugin import PluginManager from pyload.manager.Remote import RemoteManager -from pyload.manager.event.Scheduler import Scheduler -from pyload.manager.thread.Server import WebServer +from pyload.manager.Scheduler import Scheduler +from pyload.Thread.Server import WebServer from pyload.network.JsEngine import JsEngine from pyload.network.RequestFactory import RequestFactory from pyload.utils import freeSpace, formatSize @@ -364,14 +364,14 @@ class Core(object): self.lastClientConnected = 0 # later imported because they would trigger api import, and remote value not set correctly - from pyload import api + from pyload.api import Api from pyload.manager.Addon import AddonManager from pyload.manager.Thread import ThreadManager - if api.activated != self.remote: + if pyload.api.activated != self.remote: self.log.warning("Import error: API remote status not correct.") - self.api = api.Api(self) + self.api = Api(self) self.scheduler = Scheduler(self) diff --git a/pyload/Thread/Addon.py b/pyload/Thread/Addon.py new file mode 100644 index 000000000..bf3b62eb1 --- /dev/null +++ b/pyload/Thread/Addon.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +import traceback + +from Queue import Queue +from copy import copy +from os import listdir, stat +from os.path import join +from pprint import pformat +from sys import exc_info, exc_clear +from time import sleep, time, strftime, gmtime +from types import MethodType + +from pyload.Thread.Plugin import PluginThread + + +class AddonThread(PluginThread): + """thread for addons""" + + def __init__(self, m, function, args, kwargs): + """Constructor""" + PluginThread.__init__(self, m) + + self.f = function + self.args = args + self.kwargs = kwargs + + self.active = [] + + m.localThreads.append(self) + + self.start() + + + def getActiveFiles(self): + return self.active + + + def addActive(self, pyfile): + """ Adds a pyfile to active list and thus will be displayed on overview""" + if pyfile not in self.active: + self.active.append(pyfile) + + + def finishFile(self, pyfile): + if pyfile in self.active: + self.active.remove(pyfile) + + pyfile.finishIfDone() + + + def run(self): + try: + try: + self.kwargs['thread'] = self + self.f(*self.args, **self.kwargs) + except TypeError, e: + # dirty method to filter out exceptions + if "unexpected keyword argument 'thread'" not in e.args[0]: + raise + + del self.kwargs['thread'] + self.f(*self.args, **self.kwargs) + finally: + local = copy(self.active) + for x in local: + self.finishFile(x) + + self.m.localThreads.remove(self) diff --git a/pyload/Thread/Decrypter.py b/pyload/Thread/Decrypter.py new file mode 100644 index 000000000..dcb18b929 --- /dev/null +++ b/pyload/Thread/Decrypter.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +import traceback + +from Queue import Queue +from copy import copy +from os import listdir, stat +from os.path import join +from pprint import pformat +from sys import exc_info, exc_clear +from time import sleep, time, strftime, gmtime +from types import MethodType + +from pyload.Thread.Plugin import PluginThread +from pyload.plugin.Plugin import Abort, Fail, Retry + + +class DecrypterThread(PluginThread): + + """thread for decrypting""" + + def __init__(self, manager, pyfile): + """constructor""" + PluginThread.__init__(self, manager) + + self.active = pyfile + manager.localThreads.append(self) + + pyfile.setStatus("decrypting") + + self.start() + + + def getActiveFiles(self): + return [self.active] + + + def run(self): + """run method""" + + pyfile = self.active + retry = False + + try: + self.m.core.log.info(_("Decrypting starts: %s") % pyfile.name) + pyfile.error = "" + pyfile.plugin.preprocessing(self) + + except NotImplementedError: + self.m.core.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + return + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + pyfile.setStatus("offline") + self.m.core.log.warning( + _("Download is offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.m.core.log.error( + _("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + + if self.m.core.debug: + traceback.print_exc() + return + + except Abort: + self.m.core.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + if self.m.core.debug: + traceback.print_exc() + return + + except Retry: + self.m.core.log.info(_("Retrying %s") % pyfile.name) + retry = True + return self.run() + + except Exception, e: + pyfile.setStatus("failed") + self.m.core.log.error(_("Decrypting failed: %(name)s | %(msg)s") % { + "name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) + + if self.m.core.debug: + traceback.print_exc() + self.writeDebugReport(pyfile) + + return + + finally: + if not retry: + pyfile.release() + self.active = False + self.m.core.files.save() + self.m.localThreads.remove(self) + exc_clear() + + if not retry: + pyfile.delete() diff --git a/pyload/Thread/Download.py b/pyload/Thread/Download.py new file mode 100644 index 000000000..77f4f163f --- /dev/null +++ b/pyload/Thread/Download.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +import traceback + +import pycurl + +from Queue import Queue +from copy import copy +from os import listdir, stat +from os.path import join +from pprint import pformat +from sys import exc_info, exc_clear +from time import sleep, time, strftime, gmtime +from types import MethodType + +from pyload.Thread.Plugin import PluginThread +from pyload.plugin.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload + + +class DownloadThread(PluginThread): + """thread for downloading files from 'real' hoster plugins""" + + def __init__(self, manager): + """Constructor""" + PluginThread.__init__(self, manager) + + self.queue = Queue() #: job queue + self.active = False + + self.start() + + + #-------------------------------------------------------------------------- + + def run(self): + """run method""" + pyfile = None + + while True: + del pyfile + self.active = False #: sets the thread inactive when it is ready to get next job + self.active = self.queue.get() + pyfile = self.active + + if self.active == "quit": + self.active = False + self.m.threads.remove(self) + return True + + try: + if not pyfile.hasPlugin(): + continue + # this pyfile was deleted while queueing + + pyfile.plugin.checkForSameFiles(starting=True) + self.m.core.log.info(_("Download starts: %s" % pyfile.name)) + + # start download + self.m.core.addonManager.downloadPreparing(pyfile) + pyfile.error = "" + pyfile.plugin.preprocessing(self) + + self.m.core.log.info(_("Download finished: %s") % pyfile.name) + self.m.core.addonManager.downloadFinished(pyfile) + self.m.core.files.checkPackageFinished(pyfile) + + except NotImplementedError: + self.m.core.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) + pyfile.setStatus("failed") + pyfile.error = "Plugin does not work" + self.clean(pyfile) + continue + + except Abort: + try: + self.m.core.log.info(_("Download aborted: %s") % pyfile.name) + except Exception: + pass + + pyfile.setStatus("aborted") + + if self.m.core.debug: + traceback.print_exc() + + self.clean(pyfile) + continue + + except Reconnect: + self.queue.put(pyfile) + # pyfile.req.clearCookies() + + while self.m.reconnecting.isSet(): + sleep(0.5) + + continue + + except Retry, e: + reason = e.args[0] + self.m.core.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) + self.queue.put(pyfile) + continue + + except Fail, e: + msg = e.args[0] + + if msg == "offline": + pyfile.setStatus("offline") + self.m.core.log.warning(_("Download is offline: %s") % pyfile.name) + elif msg == "temp. offline": + pyfile.setStatus("temp. offline") + self.m.core.log.warning(_("Download is temporary offline: %s") % pyfile.name) + else: + pyfile.setStatus("failed") + self.m.core.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) + pyfile.error = msg + + if self.m.core.debug: + traceback.print_exc() + + self.m.core.addonManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + except pycurl.error, e: + if len(e.args) == 2: + code, msg = e.args + else: + code = 0 + msg = e.args + + self.m.core.log.debug("pycurl exception %s: %s" % (code, msg)) + + if code in (7, 18, 28, 52, 56): + self.m.core.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) + wait = time() + 60 + + pyfile.waitUntil = wait + pyfile.setStatus("waiting") + while time() < wait: + sleep(1) + if pyfile.abort: + break + + if pyfile.abort: + self.m.core.log.info(_("Download aborted: %s") % pyfile.name) + pyfile.setStatus("aborted") + + self.clean(pyfile) + else: + self.queue.put(pyfile) + + continue + + else: + pyfile.setStatus("failed") + self.m.core.log.error("pycurl error %s: %s" % (code, msg)) + if self.m.core.debug: + traceback.print_exc() + self.writeDebugReport(pyfile) + + self.m.core.addonManager.downloadFailed(pyfile) + + self.clean(pyfile) + continue + + except SkipDownload, e: + pyfile.setStatus("skipped") + + self.m.core.log.info(_("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) + + self.clean(pyfile) + + self.m.core.files.checkPackageFinished(pyfile) + + self.active = False + self.m.core.files.save() + + continue + + except Exception, e: + pyfile.setStatus("failed") + self.m.core.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) + pyfile.error = str(e) + + if self.m.core.debug: + traceback.print_exc() + self.writeDebugReport(pyfile) + + self.m.core.addonManager.downloadFailed(pyfile) + self.clean(pyfile) + continue + + finally: + self.m.core.files.save() + pyfile.checkIfProcessed() + exc_clear() + + # pyfile.plugin.req.clean() + + self.active = False + pyfile.finishIfDone() + self.m.core.files.save() + + + def put(self, job): + """assing job to thread""" + self.queue.put(job) + + + def stop(self): + """stops the thread""" + self.put("quit") diff --git a/pyload/Thread/Info.py b/pyload/Thread/Info.py new file mode 100644 index 000000000..db1b6d79c --- /dev/null +++ b/pyload/Thread/Info.py @@ -0,0 +1,221 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +import traceback + +from Queue import Queue +from copy import copy +from os import listdir, stat +from os.path import join +from pprint import pformat +from sys import exc_info, exc_clear +from time import sleep, time, strftime, gmtime +from types import MethodType + +from pyload.api import OnlineStatus +from pyload.Datatype import PyFile +from pyload.Thread.Plugin import PluginThread + + +class InfoThread(PluginThread): + + def __init__(self, manager, data, pid=-1, rid=-1, add=False): + """Constructor""" + PluginThread.__init__(self, manager) + + self.data = data + self.pid = pid #: package id + # [ .. (name, plugin) .. ] + + self.rid = rid #: result id + self.add = add #: add packages instead of return result + + self.cache = [] #: accumulated data + + self.start() + + + def run(self): + """run method""" + + plugins = {} + container = [] + + for url, plugintype, pluginname in self.data: + # filter out container plugins + if plugintype == 'container': + container.appen((pluginname, url)) + else: + if (plugintype, pluginname) in plugins: + plugins[(plugintype, pluginname)].append(url) + else: + plugins[(plugintype, pluginname)] = [url] + + # directly write to database + if self.pid > -1: + for (plugintype, pluginname), urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) + self.m.core.files.save() + + elif self.add: + for (plugintype, pluginname), urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) + + else: + # generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateCache(pluginname, result) + + packs = parseNames([(name, url) for name, x, y, url in self.cache]) + + self.m.log.debug("Fetched and generated %d packages" % len(packs)) + + for k, v in packs: + self.m.core.api.addPackage(k, v) + + # empty cache + del self.cache[:] + + else: #: post the results + + for name, url in container: + # attach container content + try: + data = self.decryptContainer(name, url) + except Exception: + traceback.print_exc() + self.m.log.error("Could not decrypt container.") + data = [] + + for url, plugintype, pluginname in data: + try: + plugins[plugintype][pluginname].append(url) + except Exception: + plugins[plugintype][pluginname] = [url] + + self.m.infoResults[self.rid] = {} + + for plugintype, pluginname, urls in plugins.iteritems(): + plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) + if hasattr(plugin, "getInfo"): + self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) + + # force to process cache + if self.cache: + self.updateResult(pluginname, [], True) + + else: + # generate default result + result = [(url, 0, 3, url) for url in urls] + + self.updateResult(pluginname, result, True) + + self.m.infoResults[self.rid]['ALL_INFO_FETCHED'] = {} + + self.m.timestamp = time() + 5 * 60 + + + def updateDB(self, plugin, result): + self.m.core.files.updateFileInfo(result, self.pid) + + + def updateResult(self, plugin, result, force=False): + # parse package name and generate result + # accumulate results + + self.cache.extend(result) + + if len(self.cache) >= 20 or force: + # used for package generating + tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) for name, size, status, url in self.cache] + + data = parseNames(tmp) + result = {} + for k, v in data.iteritems(): + for url, status in v: + status.packagename = k + result[url] = status + + self.m.setInfoResults(self.rid, result) + + self.cache = [] + + + def updateCache(self, plugin, result): + self.cache.extend(result) + + + def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): + try: + result = [] #: result loaded from cache + process = [] #: urls to process + for url in urls: + if url in self.m.infoCache: + result.append(self.m.infoCache[url]) + else: + process.append(url) + + if result: + self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) + cb(pluginname, result) + + if process: + self.m.log.debug("Run Info Fetching for %s" % pluginname) + for result in plugin.getInfo(process): + # result = [ .. (name, size, status, url) .. ] + if not type(result) == list: + result = [result] + + for res in result: + self.m.infoCache[res[3]] = res # : why don't assign res dict directly? + + cb(pluginname, result) + + self.m.log.debug("Finished Info Fetching for %s" % pluginname) + except Exception, e: + self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % {"name": pluginname, "err": str(e)}) + if self.m.core.debug: + traceback.print_exc() + + # generate default results + if err: + result = [(url, 0, 3, url) for url in urls] + cb(pluginname, result) + + + def decryptContainer(self, plugin, url): + data = [] + # only works on container plugins + + self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) + + # dummy pyfile + pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) + + pyfile.initPlugin() + + # little plugin lifecycle + try: + pyfile.plugin.setup() + pyfile.plugin.loadToDisk() + pyfile.plugin.decrypt(pyfile) + pyfile.plugin.deleteTmp() + + for pack in pyfile.plugin.packages: + pyfile.plugin.urls.extend(pack[1]) + + data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) + + self.m.log.debug("Got %d links." % len(data)) + + except Exception, e: + self.m.log.debug("Pre decrypting error: %s" % str(e)) + finally: + pyfile.release() + + return data diff --git a/pyload/Thread/Plugin.py b/pyload/Thread/Plugin.py new file mode 100644 index 000000000..f8cfba9c4 --- /dev/null +++ b/pyload/Thread/Plugin.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# @author: RaNaN + +from __future__ import with_statement + +import threading +import traceback + +from Queue import Queue +from copy import copy +from os import listdir, stat +from os.path import join +from pprint import pformat +from sys import exc_info, exc_clear +from time import sleep, time, strftime, gmtime +from types import MethodType + +from pyload.api import OnlineStatus +from pyload.Datatype import PyFile +from pyload.plugin.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload +from pyload.utils.packagetools import parseNames +from pyload.utils import fs_join + + +class PluginThread(threading.Thread): + """abstract base class for thread types""" + + def __init__(self, manager): + """Constructor""" + threading.Thread.__init__(self) + self.setDaemon(True) + self.m = manager #: thread manager + + + def writeDebugReport(self, pyfile): + """ writes a + :return: + """ + + dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) + dump = self.getDebugDump(pyfile) + + try: + import zipfile + + zip = zipfile.ZipFile(dump_name, "w") + + for f in listdir(join("tmp", pyfile.pluginname)): + try: + # avoid encoding errors + zip.write(join("tmp", pyfile.pluginname, f), fs_join(pyfile.pluginname, f)) + except Exception: + pass + + info = zipfile.ZipInfo(fs_join(pyfile.pluginname, "debug_Report.txt"), gmtime()) + info.external_attr = 0644 << 16L #: change permissions + + zip.writestr(info, dump) + zip.close() + + if not stat(dump_name).st_size: + raise Exception("Empty Zipfile") + + except Exception, e: + self.m.log.debug("Error creating zip file: %s" % e) + + dump_name = dump_name.replace(".zip", ".txt") + with open(dump_name, "wb") as f: + f.write(dump) + + self.m.core.log.info("Debug Report written to %s" % dump_name) + + + def getDebugDump(self, pyfile): + dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( + self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version, traceback.format_exc()) + + tb = exc_info()[2] + stack = [] + while tb: + stack.append(tb.tb_frame) + tb = tb.tb_next + + for frame in stack[1:]: + dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_lineno) + + for key, value in frame.f_locals.items(): + dump += "\t%20s = " % key + try: + dump += pformat(value) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + del frame + + del stack #: delete it just to be sure... + + dump += "\n\nPLUGIN OBJECT DUMP: \n\n" + + for name in dir(pyfile.plugin): + attr = getattr(pyfile.plugin, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + dump += "\nPYFILE OBJECT DUMP: \n\n" + + for name in dir(pyfile): + attr = getattr(pyfile, name) + if not name.endswith("__") and type(attr) != MethodType: + dump += "\t%20s = " % name + try: + dump += pformat(attr) + "\n" + except Exception, e: + dump += " " + str(e) + "\n" + + if pyfile.pluginname in self.m.core.config.plugin: + dump += "\n\nCONFIG: \n\n" + dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" + + return dump + + + def clean(self, pyfile): + """ set thread unactive and release pyfile """ + self.active = True #: release pyfile but lets the thread active + pyfile.release() diff --git a/pyload/Thread/Server.py b/pyload/Thread/Server.py new file mode 100644 index 000000000..6b6ef6989 --- /dev/null +++ b/pyload/Thread/Server.py @@ -0,0 +1,122 @@ +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import logging +import os +import threading + +core = None +setup = None +log = logging.getLogger("log") + + +class WebServer(threading.Thread): + + def __init__(self, pycore): + global core + + threading.Thread.__init__(self) + self.core = pycore + core = pycore + self.running = True + self.server = pycore.config.get("webui", "server") + self.https = pycore.config.get("webui", "https") + self.cert = pycore.config.get("ssl", "cert") + self.key = pycore.config.get("ssl", "key") + self.host = pycore.config.get("webui", "host") + self.port = pycore.config.get("webui", "port") + + self.setDaemon(True) + + + def run(self): + import pyload.Webui as webinterface + global webinterface + + reset = False + + if self.https and (not os.exists(self.cert) or not os.exists(self.key)): + log.warning(_("SSL certificates not found.")) + self.https = False + + if self.server in ("lighttpd", "nginx"): + log.warning(_("Sorry, we dropped support for starting %s directly within pyLoad") % self.server) + log.warning(_("You can use the threaded server which offers good performance and ssl,")) + log.warning(_("of course you can still use your existing %s with pyLoads fastcgi server") % self.server) + log.warning(_("sample configs are located in the pyload/web/servers directory")) + reset = True + elif self.server == "fastcgi": + try: + import flup + except Exception: + log.warning(_("Can't use %(server)s, python-flup is not installed!") % { + "server": self.server}) + reset = True + + if reset or self.server == "lightweight": + if os.name != "nt": + try: + import bjoern + except Exception, e: + log.error(_("Error importing lightweight server: %s") % e) + log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern")) + log.warning(_("Copy the boern.so file to lib/Python/Lib or use setup.py install")) + log.warning(_("Of course you need to be familiar with linux and know how to compile software")) + self.server = "auto" + else: + self.core.log.info(_("Server set to threaded, due to known performance problems on windows.")) + self.core.config.set("webui", "server", "threaded") + self.server = "threaded" + + if self.server == "threaded": + self.start_threaded() + elif self.server == "fastcgi": + self.start_fcgi() + elif self.server == "lightweight": + self.start_lightweight() + else: + self.start_auto() + + + def start_auto(self): + if self.https: + log.warning(_("This server offers no SSL, please consider using `threaded` instead")) + + self.core.log.info(_("Starting builtin webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_auto(host=self.host, port=self.port) + + + def start_threaded(self): + if self.https: + self.core.log.info( + _("Starting threaded SSL webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + else: + self.cert = "" + self.key = "" + self.core.log.info( + _("Starting threaded webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + + webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key) + + + def start_fcgi(self): + self.core.log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + try: + webinterface.run_fcgi(host=self.host, port=self.port) + + except ValueError: #@TODO: Fix https://github.com/pyload/pyload/issues/1145 + pass + + + def start_lightweight(self): + if self.https: + log.warning(_("This server offers no SSL, please consider using `threaded` instead")) + + self.core.log.info( + _("Starting lightweight webserver (bjoern): %(host)s:%(port)d") % {"host": self.host, "port": self.port}) + webinterface.run_lightweight(host=self.host, port=self.port) + + + def quit(self): + self.running = False diff --git a/pyload/Thread/__init__.py b/pyload/Thread/__init__.py new file mode 100644 index 000000000..30eda15c2 --- /dev/null +++ b/pyload/Thread/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +from pyload.Thread.Addon import AddonThread +from pyload.Thread.Decrypter import DecrypterThread +from pyload.Thread.Download import DownloadThread +from pyload.Thread.Info import InfoThread +from pyload.Thread.Plugin import PluginThread diff --git a/pyload/__init__.py b/pyload/__init__.py index 7f5ac0667..32235c68f 100644 --- a/pyload/__init__.py +++ b/pyload/__init__.py @@ -2,16 +2,6 @@ from __future__ import with_statement -import __builtin__ - -import os -import platform -import sys - -from codecs import getwriter - -from pyload.utils import get_console_encoding - __all__ = ["__status_code__", "__status__", "__version_info__", "__version__", "__author_name__", "__author_mail__", "__license__"] @@ -45,6 +35,9 @@ __authors__ = [("Marius" , "mkaay@mkaay.de" ), ################################# InitHomeDir ################################# +import __builtin__ +import os + __builtin__.owd = os.path.abspath("") #: original working directory __builtin__.homedir = os.path.expanduser("~") __builtin__.rootdir = os.path.abspath(os.path.join(__file__, "..")) @@ -52,14 +45,19 @@ __builtin__.configdir = "" __builtin__.pypath = os.path.abspath(os.path.join(rootdir, "..")) -if "64" in platform.machine(): - sys.path.append(os.path.join(pypath, "lib64")) +import sys + sys.path.append(os.path.join(pypath, "lib", "Python", "Lib")) sys.path.append(os.path.join(pypath, "lib")) +from codecs import getwriter + +from pyload.utils import get_console_encoding + sys.stdout = getwriter(get_console_encoding(sys.stdout.encoding))(sys.stdout, errors="replace") + if homedir == "~" and os.name == "nt": import ctypes @@ -78,6 +76,7 @@ if homedir == "~" and os.name == "nt": __builtin__.homedir = path_buf.value + try: p = os.path.join(rootdir, "config", "configdir") @@ -90,6 +89,7 @@ except IOError: else: configdir = os.path.join(homedir, "pyload") + try: if not os.path.exists(configdir): os.makedirs(configdir, 0700) diff --git a/pyload/api/__init__.py b/pyload/api/__init__.py index 442e9ef95..ddb3e16ff 100644 --- a/pyload/api/__init__.py +++ b/pyload/api/__init__.py @@ -10,7 +10,7 @@ import re from urlparse import urlparse -from pyload.datatype.File import PyFile +from pyload.Datatype import PyFile from pyload.utils.packagetools import parseNames from pyload.network.RequestFactory import getURL from pyload.remote import activated diff --git a/pyload/cli/AddPackage.py b/pyload/cli/AddPackage.py index e750274ca..3a9471cbe 100644 --- a/pyload/cli/AddPackage.py +++ b/pyload/cli/AddPackage.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # @author: RaNaN -from pyload.cli.Handler import Handler +from pyload.Cli.Handler import Handler from pyload.utils.printer import * diff --git a/pyload/cli/Cli.py b/pyload/cli/Cli.py deleted file mode 100644 index 617b65b0f..000000000 --- a/pyload/cli/Cli.py +++ /dev/null @@ -1,578 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -from __future__ import with_statement - -import os -import sys -import threading -import traceback - -import pyload.utils.pylgettext as gettext - -from codecs import getwriter -from getopt import GetoptError, getopt -from os import _exit -from os.path import join, exists, basename -from sys import exit -from time import sleep - -from Getch import Getch -from rename_process import renameProcess - -from pyload.api import Destination -from pyload.cli import AddPackage, ManageFiles -from pyload.config.Parser import ConfigParser -from pyload.remote.thriftbackend.ThriftClient import ThriftClient, NoConnection, NoSSL, WrongLogin, ConnectionClosed -from pyload.utils import formatSize, decode -from pyload.utils.printer import * - - -class Cli(object): - - def __init__(self, client, command): - self.client = client - self.command = command - - if not self.command: - renameProcess('pyload-cli') - self.getch = Getch() - self.input = "" - self.inputline = 0 - self.lastLowestLine = 0 - self.menuline = 0 - - self.lock = threading.Lock() - - # processor funcions, these will be changed dynamically depending on control flow - self.headerHandler = self #: the download status - self.bodyHandler = self #: the menu section - self.inputHandler = self - - os.system("clear") - println(1, blue("py") + yellow("Load") + white(_(" Command Line Interface"))) - println(2, "") - - self.thread = RefreshThread(self) - self.thread.start() - - self.start() - else: - self.processCommand() - - - def reset(self): - """ reset to initial main menu """ - self.input = "" - self.headerHandler = self.bodyHandler = self.inputHandler = self - - - def start(self): - """ main loop. handle input """ - while True: - # inp = raw_input() - inp = self.getch.impl() - if ord(inp) == 3: - os.system("clear") - sys.exit() #: ctrl + c - elif ord(inp) == 13: #: enter - try: - self.lock.acquire() - self.inputHandler.onEnter(self.input) - - except Exception, e: - println(2, red(e)) - finally: - self.lock.release() - - elif ord(inp) == 127: - self.input = self.input[:-1] #: backspace - try: - self.lock.acquire() - self.inputHandler.onBackSpace() - finally: - self.lock.release() - - elif ord(inp) == 27: #: ugly symbol - pass - else: - self.input += inp - try: - self.lock.acquire() - self.inputHandler.onChar(inp) - finally: - self.lock.release() - - self.inputline = self.bodyHandler.renderBody(self.menuline) - self.renderFooter(self.inputline) - - - def refresh(self): - """refresh screen""" - - println(1, blue("py") + yellow("Load") + white(_(" Command Line Interface"))) - println(2, "") - - self.lock.acquire() - - self.menuline = self.headerHandler.renderHeader(3) + 1 - println(self.menuline - 1, "") - self.inputline = self.bodyHandler.renderBody(self.menuline) - self.renderFooter(self.inputline) - - self.lock.release() - - - def setInput(self, string=""): - self.input = string - - - def setHandler(self, klass): - # create new handler with reference to cli - self.bodyHandler = self.inputHandler = klass(self) - self.input = "" - - - def renderHeader(self, line): - """ prints download status """ - # print updated information - # print "\033[J" #: clear screen - # self.println(1, blue("py") + yellow("Load") + white(_(" Command Line Interface"))) - # self.println(2, "") - # self.println(3, white(_("%s Downloads:") % (len(data)))) - - data = self.client.statusDownloads() - speed = 0 - - println(line, white(_("%s Downloads:") % (len(data)))) - line += 1 - - for download in data: - if download.status == 12: #: downloading - percent = download.percent - z = percent / 4 - speed += download.speed - println(line, cyan(download.name)) - line += 1 - println(line, - blue("[") + yellow(z * "#" + (25 - z) * " ") + blue("] ") + green(str(percent) + "%") + _( - " Speed: ") + green(formatSize(download.speed) + "/s") + _(" Size: ") + green( - download.format_size) + _(" Finished in: ") + green(download.format_eta) + _( - " ID: ") + green(download.fid)) - line += 1 - if download.status == 5: - println(line, cyan(download.name)) - line += 1 - println(line, _("waiting: ") + green(download.format_wait)) - line += 1 - - println(line, "") - line += 1 - status = self.client.statusServer() - if status.pause: - paused = _("Status:") + " " + red(_("paused")) - else: - paused = _("Status:") + " " + red(_("running")) - - println(line, "%s %s: %s %s: %s %s: %s" % ( - paused, _("total Speed"), red(formatSize(speed) + "/s"), _("Files in queue"), red( - status.queue), _("Total"), red(status.total))) - - return line + 1 - - - def renderBody(self, line): - """ prints initial menu """ - println(line, white(_("Menu:"))) - println(line + 1, "") - println(line + 2, mag("1.") + _(" Add Links")) - println(line + 3, mag("2.") + _(" Manage Queue")) - println(line + 4, mag("3.") + _(" Manage Collector")) - println(line + 5, mag("4.") + _(" (Un)Pause Server")) - println(line + 6, mag("5.") + _(" Kill Server")) - println(line + 7, mag("6.") + _(" Quit")) - - return line + 8 - - - def renderFooter(self, line): - """ prints out the input line with input """ - println(line, "") - line += 1 - - println(line, white(" Input: ") + decode(self.input)) - - # clear old output - if line < self.lastLowestLine: - for i in xrange(line + 1, self.lastLowestLine + 1): - println(i, "") - - self.lastLowestLine = line - - # set cursor to position - print "\033[" + str(self.inputline) + ";0H" - - - def onChar(self, char): - """ default no special handling for single chars """ - if char == "1": - self.setHandler(AddPackage) - elif char == "2": - self.setHandler(ManageFiles) - elif char == "3": - self.setHandler(ManageFiles) - self.bodyHandler.target = Destination.Collector - elif char == "4": - self.client.togglePause() - self.setInput() - elif char == "5": - self.client.kill() - self.client.close() - sys.exit() - elif char == "6": - os.system('clear') - sys.exit() - - - def onEnter(self, inp): - pass - - - def onBackSpace(self): - pass - - - def processCommand(self): - command = self.command[0] - args = [] - if len(self.command) > 1: - args = self.command[1:] - - if command == "status": - files = self.client.statusDownloads() - - if not files: - print "No downloads running." - - for download in files: - if download.status == 12: #: downloading - print print_status(download) - print "\tDownloading: %s @ %s/s\t %s (%s%%)" % ( - download.format_eta, formatSize(download.speed), formatSize(download.size - download.bleft), - download.percent) - elif download.status == 5: - print print_status(download) - print "\tWaiting: %s" % download.format_wait - else: - print print_status(download) - - elif command == "queue": - print_packages(self.client.getQueueData()) - - elif command == "collector": - print_packages(self.client.getCollectorData()) - - elif command == "add": - if len(args) < 2: - print _("Please use this syntax: add ...") - return - - self.client.addPackage(args[0], args[1:], Destination.Queue) - - elif command == "add_coll": - if len(args) < 2: - print _("Please use this syntax: add ...") - return - - self.client.addPackage(args[0], args[1:], Destination.Collector) - - elif command == "del_file": - self.client.deleteFiles([int(x) for x in args]) - print "Files deleted." - - elif command == "del_package": - self.client.deletePackages([int(x) for x in args]) - print "Packages deleted." - - elif command == "move": - for pid in args: - pack = self.client.getPackageInfo(int(pid)) - self.client.movePackage((pack.dest + 1) % 2, pack.pid) - - elif command == "check": - print _("Checking %d links:") % len(args) - print - rid = self.client.checkOnlineStatus(args).rid - self.printOnlineCheck(self.client, rid) - - elif command == "check_container": - path = args[0] - if not exists(join(owd, path)): - print _("File does not exists.") - return - - with open(join(owd, path), "rb") as f: - content = f.read() - - rid = self.client.checkOnlineStatusContainer([], basename(f.name), content).rid - self.printOnlineCheck(self.client, rid) - - elif command == "pause": - self.client.pause() - - elif command == "unpause": - self.client.unpause() - - elif command == "toggle": - self.client.togglePause() - - elif command == "kill": - self.client.kill() - elif command == "restart_file": - for x in args: - self.client.restartFile(int(x)) - print "Files restarted." - elif command == "restart_package": - for pid in args: - self.client.restartPackage(int(pid)) - print "Packages restarted." - - else: - print_commands() - - - def printOnlineCheck(self, client, rid): - while True: - sleep(1) - result = client.pollResults(rid) - for url, status in result.data.iteritems(): - if status.status == 2: - check = "Online" - elif status.status == 1: - check = "Offline" - else: - check = "Unknown" - - print "%-45s %-12s\t %-15s\t %s" % (status.name, formatSize(status.size), status.plugin, check) - - if result.rid == -1: - break - - -class RefreshThread(threading.Thread): - - def __init__(self, cli): - threading.Thread.__init__(self) - self.setDaemon(True) - self.cli = cli - - - def run(self): - while True: - sleep(1) - try: - self.cli.refresh() - except ConnectionClosed: - os.system("clear") - print _("pyLoad was terminated") - _exit(0) - except Exception, e: - println(2, red(str(e))) - self.cli.reset() - traceback.print_exc() - - -def print_help(config): - print - print "pyLoad CLI Copyright (c) 2008-2015 the pyLoad Team" - print - print "Usage: [python] pyload-cli.py [options] [command]" - print - print "" - print "See pyload-cli.py -c for a complete listing." - print - print "" - print " -i, --interactive", " Start in interactive mode" - print - print " -u, --username=", " " * 2, "Specify Username" - print " --pw=", " " * 2, "Password" - print " -a, --address=", " " * 3, "Specify address (current=%s)" % config['addr'] - print " -p, --port", " " * 7, "Specify port (current=%s)" % config['port'] - print - print " -l, --language", " " * 3, "Set user interface language (current=%s)" % config['language'] - print " -h, --help", " " * 7, "Display this help screen" - print " -c, --commands", " " * 3, "List all available commands" - print - - -def print_packages(data): - for pack in data: - print "Package %s (#%s):" % (pack.name, pack.pid) - for download in pack.links: - print "\t" + print_file(download) - print - - -def print_file(download): - return "#%(id)-6d %(name)-30s %(statusmsg)-10s %(plugin)-8s" % { - "id": download.fid, - "name": download.name, - "statusmsg": download.statusmsg, - "plugin": download.plugin - } - - -def print_status(download): - return "#%(id)-6s %(name)-40s Status: %(statusmsg)-10s Size: %(size)s" % { - "id": download.fid, - "name": download.name, - "statusmsg": download.statusmsg, - "size": download.format_size - } - - -def print_commands(): - commands = [("status", _("Prints server status")), - ("queue", _("Prints downloads in queue")), - ("collector", _("Prints downloads in collector")), - ("add ...", _("Adds package to queue")), - ("add_coll ...", _("Adds package to collector")), - ("del_file ...", _("Delete Files from Queue/Collector")), - ("del_package ...", _("Delete Packages from Queue/Collector")), - ("move ...", _("Move Packages from Queue to Collector or vice versa")), - ("restart_file ...", _("Restart files")), - ("restart_package ...", _("Restart packages")), - ("check ...", _("Check online status, works with local container")), - ("check_container path", _("Checks online status of a container file")), - ("pause", _("Pause the server")), - ("unpause", _("continue downloads")), - ("toggle", _("Toggle pause/unpause")), - ("kill", _("kill server")), ] - - print _("List of commands:") - print - for c in commands: - print "%-35s %s" % c - - -def writeConfig(opts): - try: - with open(join(homedir, ".pyload-cli"), "w") as cfgfile: - cfgfile.write("[cli]") - for opt in opts: - cfgfile.write("%s=%s\n" % (opt, opts[opt])) - except Exception: - print _("Couldn't write user config file") - - -def main(): - config = {"addr": "127.0.0.1", "port": "7227", "language": "en"} - try: - config['language'] = os.environ['LANG'][0:2] - except Exception: - pass - - if (not exists(join(pypath, "locale", config['language']))) or config['language'] == "": - config['language'] = "en" - - configFile = ConfigParser.ConfigParser() - configFile.read(join(homedir, ".pyload-cli")) - - if configFile.has_section("cli"): - for opt in configFile.items("cli"): - config[opt[0]] = opt[1] - - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("Cli", join(pypath, "locale"), - languages=[config['language'], "en"], fallback=True) - translation.install(unicode=True) - - interactive = False - command = None - username = "" - password = "" - - shortOptions = 'iu:p:a:hcl:' - longOptions = ['interactive', "username=", "pw=", "address=", "port=", "help", "commands", "language="] - - try: - opts, extraparams = getopt(sys.argv[1:], shortOptions, longOptions) - for option, params in opts: - if option in ("-i", "--interactive"): - interactive = True - elif option in ("-u", "--username"): - username = params - elif option in ("-a", "--address"): - config['addr'] = params - elif option in ("-p", "--port"): - config['port'] = params - elif option in ("-l", "--language"): - config['language'] = params - gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) - translation = gettext.translation("Cli", join(pypath, "locale"), - languages=[config['language'], "en"], fallback=True) - translation.install(unicode=True) - elif option in ("-h", "--help"): - print_help(config) - exit() - elif option in ("--pw"): - password = params - elif option in ("-c", "--comands"): - print_commands() - exit() - - except GetoptError: - print 'Unknown Argument(s) "%s"' % " ".join(sys.argv[1:]) - print_help(config) - exit() - - if len(extraparams) >= 1: - command = extraparams - - client = False - - if interactive: - try: - client = ThriftClient(config['addr'], int(config['port']), username, password) - except WrongLogin: - pass - except NoSSL: - print _("You need py-openssl to connect to this pyLoad Core.") - exit() - except NoConnection: - config['addr'] = False - config['port'] = False - - if not client: - if not config['addr']: config['addr'] = raw_input(_("Address: ")) - if not config['port']: config['port'] = raw_input(_("Port: ")) - if not username: username = raw_input(_("Username: ")) - if not password: - from getpass import getpass - - password = getpass(_("Password: ")) - - try: - client = ThriftClient(config['addr'], int(config['port']), username, password) - except WrongLogin: - print _("Login data is wrong.") - except NoConnection: - print _("Could not establish connection to %(addr)s:%(port)s." % {"addr": config['addr'], - "port": config['port']}) - - else: - try: - client = ThriftClient(config['addr'], int(config['port']), username, password) - except WrongLogin: - print _("Login data is wrong.") - except NoConnection: - print _("Could not establish connection to %(addr)s:%(port)s." % {"addr": config['addr'], - "port": config['port']}) - except NoSSL: - print _("You need py-openssl to connect to this pyLoad core.") - - if interactive and command: print _("Interactive mode ignored since you passed some commands.") - - if client: - writeConfig(config) - cli = Cli(client, command) diff --git a/pyload/cli/ManageFiles.py b/pyload/cli/ManageFiles.py index 3833b2c48..017529852 100644 --- a/pyload/cli/ManageFiles.py +++ b/pyload/cli/ManageFiles.py @@ -4,7 +4,7 @@ from itertools import islice from time import time -from pyload.cli.Handler import Handler +from pyload.Cli.Handler import Handler from pyload.utils.printer import * from pyload.api import Destination, PackageData diff --git a/pyload/cli/__init__.py b/pyload/cli/__init__.py index a64fc0c0c..3449109ba 100644 --- a/pyload/cli/__init__.py +++ b/pyload/cli/__init__.py @@ -1,4 +1,583 @@ # -*- coding: utf-8 -*- +# @author: RaNaN -from pyload.cli.AddPackage import AddPackage -from pyload.cli.ManageFiles import ManageFiles +from __future__ import with_statement + +import os +import sys +import threading +import traceback + +import pyload.utils.pylgettext as gettext + +from codecs import getwriter +from getopt import GetoptError, getopt +from os import _exit +from os.path import join, exists, basename +from sys import exit +from time import sleep + +from Getch import Getch +from rename_process import renameProcess + +from pyload.api import Destination +from pyload.Cli.AddPackage import AddPackage +from pyload.Cli.ManageFiles import ManageFiles +from pyload.config.Parser import ConfigParser +from pyload.remote.thriftbackend.ThriftClient import ThriftClient, NoConnection, NoSSL, WrongLogin, ConnectionClosed +from pyload.utils import formatSize, decode +from pyload.utils.printer import * + + +class Cli(object): + + def __init__(self, client, command): + self.client = client + self.command = command + + if not self.command: + renameProcess('pyload-cli') + self.getch = Getch() + self.input = "" + self.inputline = 0 + self.lastLowestLine = 0 + self.menuline = 0 + + self.lock = threading.Lock() + + # processor funcions, these will be changed dynamically depending on control flow + self.headerHandler = self #: the download status + self.bodyHandler = self #: the menu section + self.inputHandler = self + + os.system("clear") + println(1, blue("py") + yellow("Load") + white(_(" Command Line Interface"))) + println(2, "") + + self.thread = RefreshThread(self) + self.thread.start() + + self.start() + else: + self.processCommand() + + + def reset(self): + """ reset to initial main menu """ + self.input = "" + self.headerHandler = self.bodyHandler = self.inputHandler = self + + + def start(self): + """ main loop. handle input """ + while True: + # inp = raw_input() + inp = self.getch.impl() + if ord(inp) == 3: + os.system("clear") + sys.exit() #: ctrl + c + elif ord(inp) == 13: #: enter + try: + self.lock.acquire() + self.inputHandler.onEnter(self.input) + + except Exception, e: + println(2, red(e)) + finally: + self.lock.release() + + elif ord(inp) == 127: + self.input = self.input[:-1] #: backspace + try: + self.lock.acquire() + self.inputHandler.onBackSpace() + finally: + self.lock.release() + + elif ord(inp) == 27: #: ugly symbol + pass + else: + self.input += inp + try: + self.lock.acquire() + self.inputHandler.onChar(inp) + finally: + self.lock.release() + + self.inputline = self.bodyHandler.renderBody(self.menuline) + self.renderFooter(self.inputline) + + + def refresh(self): + """refresh screen""" + + println(1, blue("py") + yellow("Load") + white(_(" Command Line Interface"))) + println(2, "") + + self.lock.acquire() + + self.menuline = self.headerHandler.renderHeader(3) + 1 + println(self.menuline - 1, "") + self.inputline = self.bodyHandler.renderBody(self.menuline) + self.renderFooter(self.inputline) + + self.lock.release() + + + def setInput(self, string=""): + self.input = string + + + def setHandler(self, klass): + # create new handler with reference to cli + self.bodyHandler = self.inputHandler = klass(self) + self.input = "" + + + def renderHeader(self, line): + """ prints download status """ + # print updated information + # print "\033[J" #: clear screen + # self.println(1, blue("py") + yellow("Load") + white(_(" Command Line Interface"))) + # self.println(2, "") + # self.println(3, white(_("%s Downloads:") % (len(data)))) + + data = self.client.statusDownloads() + speed = 0 + + println(line, white(_("%s Downloads:") % (len(data)))) + line += 1 + + for download in data: + if download.status == 12: #: downloading + percent = download.percent + z = percent / 4 + speed += download.speed + println(line, cyan(download.name)) + line += 1 + println(line, + blue("[") + yellow(z * "#" + (25 - z) * " ") + blue("] ") + green(str(percent) + "%") + _( + " Speed: ") + green(formatSize(download.speed) + "/s") + _(" Size: ") + green( + download.format_size) + _(" Finished in: ") + green(download.format_eta) + _( + " ID: ") + green(download.fid)) + line += 1 + if download.status == 5: + println(line, cyan(download.name)) + line += 1 + println(line, _("waiting: ") + green(download.format_wait)) + line += 1 + + println(line, "") + line += 1 + status = self.client.statusServer() + if status.pause: + paused = _("Status:") + " " + red(_("paused")) + else: + paused = _("Status:") + " " + red(_("running")) + + println(line, "%s %s: %s %s: %s %s: %s" % ( + paused, _("total Speed"), red(formatSize(speed) + "/s"), _("Files in queue"), red( + status.queue), _("Total"), red(status.total))) + + return line + 1 + + + def renderBody(self, line): + """ prints initial menu """ + println(line, white(_("Menu:"))) + println(line + 1, "") + println(line + 2, mag("1.") + _(" Add Links")) + println(line + 3, mag("2.") + _(" Manage Queue")) + println(line + 4, mag("3.") + _(" Manage Collector")) + println(line + 5, mag("4.") + _(" (Un)Pause Server")) + println(line + 6, mag("5.") + _(" Kill Server")) + println(line + 7, mag("6.") + _(" Quit")) + + return line + 8 + + + def renderFooter(self, line): + """ prints out the input line with input """ + println(line, "") + line += 1 + + println(line, white(" Input: ") + decode(self.input)) + + # clear old output + if line < self.lastLowestLine: + for i in xrange(line + 1, self.lastLowestLine + 1): + println(i, "") + + self.lastLowestLine = line + + # set cursor to position + print "\033[" + str(self.inputline) + ";0H" + + + def onChar(self, char): + """ default no special handling for single chars """ + if char == "1": + self.setHandler(AddPackage) + elif char == "2": + self.setHandler(ManageFiles) + elif char == "3": + self.setHandler(ManageFiles) + self.bodyHandler.target = Destination.Collector + elif char == "4": + self.client.togglePause() + self.setInput() + elif char == "5": + self.client.kill() + self.client.close() + sys.exit() + elif char == "6": + os.system('clear') + sys.exit() + + + def onEnter(self, inp): + pass + + + def onBackSpace(self): + pass + + + def processCommand(self): + command = self.command[0] + args = [] + if len(self.command) > 1: + args = self.command[1:] + + if command == "status": + files = self.client.statusDownloads() + + if not files: + print "No downloads running." + + for download in files: + if download.status == 12: #: downloading + print print_status(download) + print "\tDownloading: %s @ %s/s\t %s (%s%%)" % ( + download.format_eta, formatSize(download.speed), formatSize(download.size - download.bleft), + download.percent) + elif download.status == 5: + print print_status(download) + print "\tWaiting: %s" % download.format_wait + else: + print print_status(download) + + elif command == "queue": + print_packages(self.client.getQueueData()) + + elif command == "collector": + print_packages(self.client.getCollectorData()) + + elif command == "add": + if len(args) < 2: + print _("Please use this syntax: add ...") + return + + self.client.addPackage(args[0], args[1:], Destination.Queue) + + elif command == "add_coll": + if len(args) < 2: + print _("Please use this syntax: add ...") + return + + self.client.addPackage(args[0], args[1:], Destination.Collector) + + elif command == "del_file": + self.client.deleteFiles([int(x) for x in args]) + print "Files deleted." + + elif command == "del_package": + self.client.deletePackages([int(x) for x in args]) + print "Packages deleted." + + elif command == "move": + for pid in args: + pack = self.client.getPackageInfo(int(pid)) + self.client.movePackage((pack.dest + 1) % 2, pack.pid) + + elif command == "check": + print _("Checking %d links:") % len(args) + print + rid = self.client.checkOnlineStatus(args).rid + self.printOnlineCheck(self.client, rid) + + elif command == "check_container": + path = args[0] + if not exists(join(owd, path)): + print _("File does not exists.") + return + + with open(join(owd, path), "rb") as f: + content = f.read() + + rid = self.client.checkOnlineStatusContainer([], basename(f.name), content).rid + self.printOnlineCheck(self.client, rid) + + elif command == "pause": + self.client.pause() + + elif command == "unpause": + self.client.unpause() + + elif command == "toggle": + self.client.togglePause() + + elif command == "kill": + self.client.kill() + elif command == "restart_file": + for x in args: + self.client.restartFile(int(x)) + print "Files restarted." + elif command == "restart_package": + for pid in args: + self.client.restartPackage(int(pid)) + print "Packages restarted." + + else: + print_commands() + + + def printOnlineCheck(self, client, rid): + while True: + sleep(1) + result = client.pollResults(rid) + for url, status in result.data.iteritems(): + if status.status == 2: + check = "Online" + elif status.status == 1: + check = "Offline" + else: + check = "Unknown" + + print "%-45s %-12s\t %-15s\t %s" % (status.name, formatSize(status.size), status.plugin, check) + + if result.rid == -1: + break + + +class RefreshThread(threading.Thread): + + def __init__(self, cli): + threading.Thread.__init__(self) + self.setDaemon(True) + self.cli = cli + + + def run(self): + while True: + sleep(1) + try: + self.cli.refresh() + except ConnectionClosed: + os.system("clear") + print _("pyLoad was terminated") + _exit(0) + except Exception, e: + println(2, red(str(e))) + self.cli.reset() + traceback.print_exc() + + +def print_help(config): + print + print "pyLoad CLI Copyright (c) 2008-2015 the pyLoad Team" + print + print "Usage: [python] pyload-cli.py [options] [command]" + print + print "" + print "See pyload-cli.py -c for a complete listing." + print + print "" + print " -i, --interactive", " Start in interactive mode" + print + print " -u, --username=", " " * 2, "Specify Username" + print " --pw=", " " * 2, "Password" + print " -a, --address=", " " * 3, "Specify address (current=%s)" % config['addr'] + print " -p, --port", " " * 7, "Specify port (current=%s)" % config['port'] + print + print " -l, --language", " " * 3, "Set user interface language (current=%s)" % config['language'] + print " -h, --help", " " * 7, "Display this help screen" + print " -c, --commands", " " * 3, "List all available commands" + print + + +def print_packages(data): + for pack in data: + print "Package %s (#%s):" % (pack.name, pack.pid) + for download in pack.links: + print "\t" + print_file(download) + print + + +def print_file(download): + return "#%(id)-6d %(name)-30s %(statusmsg)-10s %(plugin)-8s" % { + "id": download.fid, + "name": download.name, + "statusmsg": download.statusmsg, + "plugin": download.plugin + } + + +def print_status(download): + return "#%(id)-6s %(name)-40s Status: %(statusmsg)-10s Size: %(size)s" % { + "id": download.fid, + "name": download.name, + "statusmsg": download.statusmsg, + "size": download.format_size + } + + +def print_commands(): + commands = [("status", _("Prints server status")), + ("queue", _("Prints downloads in queue")), + ("collector", _("Prints downloads in collector")), + ("add ...", _("Adds package to queue")), + ("add_coll ...", _("Adds package to collector")), + ("del_file ...", _("Delete Files from Queue/Collector")), + ("del_package ...", _("Delete Packages from Queue/Collector")), + ("move ...", _("Move Packages from Queue to Collector or vice versa")), + ("restart_file ...", _("Restart files")), + ("restart_package ...", _("Restart packages")), + ("check ...", _("Check online status, works with local container")), + ("check_container path", _("Checks online status of a container file")), + ("pause", _("Pause the server")), + ("unpause", _("continue downloads")), + ("toggle", _("Toggle pause/unpause")), + ("kill", _("kill server")), ] + + print _("List of commands:") + print + for c in commands: + print "%-35s %s" % c + + +def writeConfig(opts): + try: + with open(join(homedir, ".pyload-cli"), "w") as cfgfile: + cfgfile.write("[cli]") + for opt in opts: + cfgfile.write("%s=%s\n" % (opt, opts[opt])) + except Exception: + print _("Couldn't write user config file") + + +def main(): + config = {"addr": "127.0.0.1", "port": "7227", "language": "en"} + try: + config['language'] = os.environ['LANG'][0:2] + except Exception: + pass + + if (not exists(join(pypath, "locale", config['language']))) or config['language'] == "": + config['language'] = "en" + + configFile = ConfigParser.ConfigParser() + configFile.read(join(homedir, ".pyload-cli")) + + if configFile.has_section("cli"): + for opt in configFile.items("cli"): + config[opt[0]] = opt[1] + + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("Cli", join(pypath, "locale"), + languages=[config['language'], "en"], fallback=True) + translation.install(unicode=True) + + interactive = False + command = None + username = "" + password = "" + + shortOptions = 'iu:p:a:hcl:' + longOptions = ['interactive', "username=", "pw=", "address=", "port=", "help", "commands", "language="] + + try: + opts, extraparams = getopt(sys.argv[1:], shortOptions, longOptions) + for option, params in opts: + if option in ("-i", "--interactive"): + interactive = True + elif option in ("-u", "--username"): + username = params + elif option in ("-a", "--address"): + config['addr'] = params + elif option in ("-p", "--port"): + config['port'] = params + elif option in ("-l", "--language"): + config['language'] = params + gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) + translation = gettext.translation("Cli", join(pypath, "locale"), + languages=[config['language'], "en"], fallback=True) + translation.install(unicode=True) + elif option in ("-h", "--help"): + print_help(config) + exit() + elif option in ("--pw"): + password = params + elif option in ("-c", "--comands"): + print_commands() + exit() + + except GetoptError: + print 'Unknown Argument(s) "%s"' % " ".join(sys.argv[1:]) + print_help(config) + exit() + + if len(extraparams) >= 1: + command = extraparams + + client = False + + if interactive: + try: + client = ThriftClient(config['addr'], int(config['port']), username, password) + except WrongLogin: + pass + except NoSSL: + print _("You need py-openssl to connect to this pyLoad Core.") + exit() + except NoConnection: + config['addr'] = False + config['port'] = False + + if not client: + if not config['addr']: config['addr'] = raw_input(_("Address: ")) + if not config['port']: config['port'] = raw_input(_("Port: ")) + if not username: username = raw_input(_("Username: ")) + if not password: + from getpass import getpass + + password = getpass(_("Password: ")) + + try: + client = ThriftClient(config['addr'], int(config['port']), username, password) + except WrongLogin: + print _("Login data is wrong.") + except NoConnection: + print _("Could not establish connection to %(addr)s:%(port)s." % {"addr": config['addr'], + "port": config['port']}) + + else: + try: + client = ThriftClient(config['addr'], int(config['port']), username, password) + except WrongLogin: + print _("Login data is wrong.") + except NoConnection: + print _("Could not establish connection to %(addr)s:%(port)s." % {"addr": config['addr'], + "port": config['port']}) + except NoSSL: + print _("You need py-openssl to connect to this pyLoad core.") + + if interactive and command: print _("Interactive mode ignored since you passed some commands.") + + if client: + writeConfig(config) + cli = Cli(client, command) + + +if __name__ == "__main__": + main() diff --git a/pyload/config/Setup.py b/pyload/config/Setup.py index e2f96c802..2143632c2 100644 --- a/pyload/config/Setup.py +++ b/pyload/config/Setup.py @@ -295,7 +295,7 @@ class SetupAssistant(object): print print _("The following logindata is valid for CLI and webinterface.") - from pyload.database import DatabaseBackend + from pyload.Database import DatabaseBackend db = DatabaseBackend(None) db.setup() @@ -375,7 +375,7 @@ class SetupAssistant(object): def set_user(self): load_translation("setup", self.config.get("general", "language")) - from pyload.database import DatabaseBackend + from pyload.Database import DatabaseBackend db = DatabaseBackend(None) db.setup() diff --git a/pyload/database/File.py b/pyload/database/File.py index db4b13a52..f28535ffa 100644 --- a/pyload/database/File.py +++ b/pyload/database/File.py @@ -3,11 +3,10 @@ import threading -from pyload.utils import formatSize, lock +from pyload.Datatype import PyFile, PyPackage +from pyload.Database import DatabaseBackend, style from pyload.manager.Event import InsertEvent, ReloadAllEvent, RemoveEvent, UpdateEvent -from pyload.datatype.Package import PyPackage -from pyload.datatype.File import PyFile -from pyload.database import DatabaseBackend, style +from pyload.utils import formatSize, lock try: from pysqlite2 import dbapi2 as sqlite3 diff --git a/pyload/database/Storage.py b/pyload/database/Storage.py index 70932b55c..a19f67606 100644 --- a/pyload/database/Storage.py +++ b/pyload/database/Storage.py @@ -1,13 +1,12 @@ # -*- coding: utf-8 -*- # @author: mkaay -from pyload.database import style -from pyload.database import DatabaseBackend +from pyload.Database import style +from pyload.Database import DatabaseBackend class StorageMethods(object): - @style.queue def setStorage(db, identifier, key, value): db.c.execute("SELECT id FROM storage WHERE identifier=? AND key=?", (identifier, key)) diff --git a/pyload/database/User.py b/pyload/database/User.py index b5b44c50a..dc60ce23a 100644 --- a/pyload/database/User.py +++ b/pyload/database/User.py @@ -4,7 +4,7 @@ from hashlib import sha1 import random -from pyload.database import DatabaseBackend, style +from pyload.Database import DatabaseBackend, style class UserMethods(object): diff --git a/pyload/database/__init__.py b/pyload/database/__init__.py index 4e0edc5d1..b4200a698 100644 --- a/pyload/database/__init__.py +++ b/pyload/database/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from pyload.database.Backend import DatabaseBackend, style -from pyload.database.File import FileHandler -from pyload.database.Storage import StorageMethods -from pyload.database.User import UserMethods +from pyload.Database.Backend import DatabaseBackend, style +from pyload.Database.File import FileHandler +from pyload.Database.Storage import StorageMethods +from pyload.Database.User import UserMethods diff --git a/pyload/datatype/__init__.py b/pyload/datatype/__init__.py index 40a96afc6..29f0d40aa 100644 --- a/pyload/datatype/__init__.py +++ b/pyload/datatype/__init__.py @@ -1 +1,4 @@ # -*- coding: utf-8 -*- + +from pyload.Datatype.File import PyFile +from pyload.Datatype.Package import PyPackage diff --git a/pyload/manager/Addon.py b/pyload/manager/Addon.py index 2b3d1c456..98b66189a 100644 --- a/pyload/manager/Addon.py +++ b/pyload/manager/Addon.py @@ -9,7 +9,7 @@ import traceback from types import MethodType -from pyload.manager.thread.Addon import AddonThread +from pyload.Thread import AddonThread from pyload.manager.Plugin import literal_eval from pyload.utils import lock diff --git a/pyload/manager/Scheduler.py b/pyload/manager/Scheduler.py new file mode 100644 index 000000000..d7098ae10 --- /dev/null +++ b/pyload/manager/Scheduler.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- +# @author: mkaay + +import threading + +from heapq import heappop, heappush +from time import time + + +class AlreadyCalled(Exception): + pass + + +class Deferred(object): + + def __init__(self): + self.call = [] + self.result = () + + + def addCallback(self, f, *cargs, **ckwargs): + self.call.append((f, cargs, ckwargs)) + + + def callback(self, *args, **kwargs): + if self.result: + raise AlreadyCalled + self.result = (args, kwargs) + for f, cargs, ckwargs in self.call: + args += tuple(cargs) + kwargs.update(ckwargs) + f(*args ** kwargs) + + +class Scheduler(object): + + def __init__(self, core): + self.core = core + + self.queue = PriorityQueue() + + + def addJob(self, t, call, args=[], kwargs={}, threaded=True): + d = Deferred() + t += time() + j = Job(t, call, args, kwargs, d, threaded) + self.queue.put((t, j)) + return d + + + def removeJob(self, d): + """ + :param d: defered object + :return: if job was deleted + """ + index = -1 + + for i, j in enumerate(self.queue): + if j[1].deferred == d: + index = i + + if index >= 0: + del self.queue[index] + return True + + return False + + + def work(self): + while True: + t, j = self.queue.get() + if not j: + break + else: + if t <= time(): + j.start() + else: + self.queue.put((t, j)) + break + + +class Job(object): + + def __init__(self, time, call, args=[], kwargs={}, deferred=None, threaded=True): + self.time = float(time) + self.call = call + self.args = args + self.kwargs = kwargs + self.deferred = deferred + self.threaded = threaded + + + def run(self): + ret = self.call(*self.args, **self.kwargs) + if self.deferred is None: + return + else: + self.deferred.callback(ret) + + + def start(self): + if self.threaded: + t = threading.Thread(target=self.run) + t.setDaemon(True) + t.start() + else: + self.run() + + +class PriorityQueue(object): + """ a non blocking priority queue """ + + def __init__(self): + self.queue = [] + self.lock = threading.Lock() + + + def __iter__(self): + return iter(self.queue) + + + def __delitem__(self, key): + del self.queue[key] + + + def put(self, element): + self.lock.acquire() + heappush(self.queue, element) + self.lock.release() + + + def get(self): + """ return element or None """ + self.lock.acquire() + try: + el = heappop(self.queue) + return el + except IndexError: + return None, None + finally: + self.lock.release() diff --git a/pyload/manager/Thread.py b/pyload/manager/Thread.py index ecfcb3e26..6df8ab622 100644 --- a/pyload/manager/Thread.py +++ b/pyload/manager/Thread.py @@ -13,10 +13,8 @@ from random import choice from subprocess import Popen from time import sleep, time -from pyload.datatype.File import PyFile -from pyload.manager.thread.Decrypter import DecrypterThread -from pyload.manager.thread.Download import DownloadThread -from pyload.manager.thread.Info import InfoThread +from pyload.Datatype import PyFile +from pyload.Thread import DecrypterThread, DownloadThread, InfoThread from pyload.network.RequestFactory import getURL from pyload.utils import freeSpace, lock diff --git a/pyload/manager/event/Scheduler.py b/pyload/manager/event/Scheduler.py deleted file mode 100644 index d7098ae10..000000000 --- a/pyload/manager/event/Scheduler.py +++ /dev/null @@ -1,141 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: mkaay - -import threading - -from heapq import heappop, heappush -from time import time - - -class AlreadyCalled(Exception): - pass - - -class Deferred(object): - - def __init__(self): - self.call = [] - self.result = () - - - def addCallback(self, f, *cargs, **ckwargs): - self.call.append((f, cargs, ckwargs)) - - - def callback(self, *args, **kwargs): - if self.result: - raise AlreadyCalled - self.result = (args, kwargs) - for f, cargs, ckwargs in self.call: - args += tuple(cargs) - kwargs.update(ckwargs) - f(*args ** kwargs) - - -class Scheduler(object): - - def __init__(self, core): - self.core = core - - self.queue = PriorityQueue() - - - def addJob(self, t, call, args=[], kwargs={}, threaded=True): - d = Deferred() - t += time() - j = Job(t, call, args, kwargs, d, threaded) - self.queue.put((t, j)) - return d - - - def removeJob(self, d): - """ - :param d: defered object - :return: if job was deleted - """ - index = -1 - - for i, j in enumerate(self.queue): - if j[1].deferred == d: - index = i - - if index >= 0: - del self.queue[index] - return True - - return False - - - def work(self): - while True: - t, j = self.queue.get() - if not j: - break - else: - if t <= time(): - j.start() - else: - self.queue.put((t, j)) - break - - -class Job(object): - - def __init__(self, time, call, args=[], kwargs={}, deferred=None, threaded=True): - self.time = float(time) - self.call = call - self.args = args - self.kwargs = kwargs - self.deferred = deferred - self.threaded = threaded - - - def run(self): - ret = self.call(*self.args, **self.kwargs) - if self.deferred is None: - return - else: - self.deferred.callback(ret) - - - def start(self): - if self.threaded: - t = threading.Thread(target=self.run) - t.setDaemon(True) - t.start() - else: - self.run() - - -class PriorityQueue(object): - """ a non blocking priority queue """ - - def __init__(self): - self.queue = [] - self.lock = threading.Lock() - - - def __iter__(self): - return iter(self.queue) - - - def __delitem__(self, key): - del self.queue[key] - - - def put(self, element): - self.lock.acquire() - heappush(self.queue, element) - self.lock.release() - - - def get(self): - """ return element or None """ - self.lock.acquire() - try: - el = heappop(self.queue) - return el - except IndexError: - return None, None - finally: - self.lock.release() diff --git a/pyload/manager/event/__init__.py b/pyload/manager/event/__init__.py deleted file mode 100644 index 40a96afc6..000000000 --- a/pyload/manager/event/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- diff --git a/pyload/manager/thread/Addon.py b/pyload/manager/thread/Addon.py deleted file mode 100644 index 3cda99950..000000000 --- a/pyload/manager/thread/Addon.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -import traceback - -from Queue import Queue -from copy import copy -from os import listdir, stat -from os.path import join -from pprint import pformat -from sys import exc_info, exc_clear -from time import sleep, time, strftime, gmtime -from types import MethodType - -from pyload.manager.thread.Plugin import PluginThread - - -class AddonThread(PluginThread): - """thread for addons""" - - def __init__(self, m, function, args, kwargs): - """Constructor""" - PluginThread.__init__(self, m) - - self.f = function - self.args = args - self.kwargs = kwargs - - self.active = [] - - m.localThreads.append(self) - - self.start() - - - def getActiveFiles(self): - return self.active - - - def addActive(self, pyfile): - """ Adds a pyfile to active list and thus will be displayed on overview""" - if pyfile not in self.active: - self.active.append(pyfile) - - - def finishFile(self, pyfile): - if pyfile in self.active: - self.active.remove(pyfile) - - pyfile.finishIfDone() - - - def run(self): - try: - try: - self.kwargs['thread'] = self - self.f(*self.args, **self.kwargs) - except TypeError, e: - # dirty method to filter out exceptions - if "unexpected keyword argument 'thread'" not in e.args[0]: - raise - - del self.kwargs['thread'] - self.f(*self.args, **self.kwargs) - finally: - local = copy(self.active) - for x in local: - self.finishFile(x) - - self.m.localThreads.remove(self) diff --git a/pyload/manager/thread/Decrypter.py b/pyload/manager/thread/Decrypter.py deleted file mode 100644 index 3554feac4..000000000 --- a/pyload/manager/thread/Decrypter.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -import traceback - -from Queue import Queue -from copy import copy -from os import listdir, stat -from os.path import join -from pprint import pformat -from sys import exc_info, exc_clear -from time import sleep, time, strftime, gmtime -from types import MethodType - -from pyload.manager.thread.Plugin import PluginThread -from pyload.plugin.Plugin import Abort, Fail, Retry - - -class DecrypterThread(PluginThread): - - """thread for decrypting""" - - def __init__(self, manager, pyfile): - """constructor""" - PluginThread.__init__(self, manager) - - self.active = pyfile - manager.localThreads.append(self) - - pyfile.setStatus("decrypting") - - self.start() - - - def getActiveFiles(self): - return [self.active] - - - def run(self): - """run method""" - - pyfile = self.active - retry = False - - try: - self.m.core.log.info(_("Decrypting starts: %s") % pyfile.name) - pyfile.error = "" - pyfile.plugin.preprocessing(self) - - except NotImplementedError: - self.m.core.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) - return - - except Fail, e: - msg = e.args[0] - - if msg == "offline": - pyfile.setStatus("offline") - self.m.core.log.warning( - _("Download is offline: %s") % pyfile.name) - else: - pyfile.setStatus("failed") - self.m.core.log.error( - _("Decrypting failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) - pyfile.error = msg - - if self.m.core.debug: - traceback.print_exc() - return - - except Abort: - self.m.core.log.info(_("Download aborted: %s") % pyfile.name) - pyfile.setStatus("aborted") - - if self.m.core.debug: - traceback.print_exc() - return - - except Retry: - self.m.core.log.info(_("Retrying %s") % pyfile.name) - retry = True - return self.run() - - except Exception, e: - pyfile.setStatus("failed") - self.m.core.log.error(_("Decrypting failed: %(name)s | %(msg)s") % { - "name": pyfile.name, "msg": str(e)}) - pyfile.error = str(e) - - if self.m.core.debug: - traceback.print_exc() - self.writeDebugReport(pyfile) - - return - - finally: - if not retry: - pyfile.release() - self.active = False - self.m.core.files.save() - self.m.localThreads.remove(self) - exc_clear() - - if not retry: - pyfile.delete() diff --git a/pyload/manager/thread/Download.py b/pyload/manager/thread/Download.py deleted file mode 100644 index 04f73b2ed..000000000 --- a/pyload/manager/thread/Download.py +++ /dev/null @@ -1,213 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -import traceback - -import pycurl - -from Queue import Queue -from copy import copy -from os import listdir, stat -from os.path import join -from pprint import pformat -from sys import exc_info, exc_clear -from time import sleep, time, strftime, gmtime -from types import MethodType - -from pyload.manager.thread.Plugin import PluginThread -from pyload.plugin.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload - - -class DownloadThread(PluginThread): - """thread for downloading files from 'real' hoster plugins""" - - def __init__(self, manager): - """Constructor""" - PluginThread.__init__(self, manager) - - self.queue = Queue() #: job queue - self.active = False - - self.start() - - - #-------------------------------------------------------------------------- - - def run(self): - """run method""" - pyfile = None - - while True: - del pyfile - self.active = False #: sets the thread inactive when it is ready to get next job - self.active = self.queue.get() - pyfile = self.active - - if self.active == "quit": - self.active = False - self.m.threads.remove(self) - return True - - try: - if not pyfile.hasPlugin(): - continue - # this pyfile was deleted while queueing - - pyfile.plugin.checkForSameFiles(starting=True) - self.m.core.log.info(_("Download starts: %s" % pyfile.name)) - - # start download - self.m.core.addonManager.downloadPreparing(pyfile) - pyfile.error = "" - pyfile.plugin.preprocessing(self) - - self.m.core.log.info(_("Download finished: %s") % pyfile.name) - self.m.core.addonManager.downloadFinished(pyfile) - self.m.core.files.checkPackageFinished(pyfile) - - except NotImplementedError: - self.m.core.log.error(_("Plugin %s is missing a function.") % pyfile.pluginname) - pyfile.setStatus("failed") - pyfile.error = "Plugin does not work" - self.clean(pyfile) - continue - - except Abort: - try: - self.m.core.log.info(_("Download aborted: %s") % pyfile.name) - except Exception: - pass - - pyfile.setStatus("aborted") - - if self.m.core.debug: - traceback.print_exc() - - self.clean(pyfile) - continue - - except Reconnect: - self.queue.put(pyfile) - # pyfile.req.clearCookies() - - while self.m.reconnecting.isSet(): - sleep(0.5) - - continue - - except Retry, e: - reason = e.args[0] - self.m.core.log.info(_("Download restarted: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": reason}) - self.queue.put(pyfile) - continue - - except Fail, e: - msg = e.args[0] - - if msg == "offline": - pyfile.setStatus("offline") - self.m.core.log.warning(_("Download is offline: %s") % pyfile.name) - elif msg == "temp. offline": - pyfile.setStatus("temp. offline") - self.m.core.log.warning(_("Download is temporary offline: %s") % pyfile.name) - else: - pyfile.setStatus("failed") - self.m.core.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": msg}) - pyfile.error = msg - - if self.m.core.debug: - traceback.print_exc() - - self.m.core.addonManager.downloadFailed(pyfile) - self.clean(pyfile) - continue - - except pycurl.error, e: - if len(e.args) == 2: - code, msg = e.args - else: - code = 0 - msg = e.args - - self.m.core.log.debug("pycurl exception %s: %s" % (code, msg)) - - if code in (7, 18, 28, 52, 56): - self.m.core.log.warning(_("Couldn't connect to host or connection reset, waiting 1 minute and retry.")) - wait = time() + 60 - - pyfile.waitUntil = wait - pyfile.setStatus("waiting") - while time() < wait: - sleep(1) - if pyfile.abort: - break - - if pyfile.abort: - self.m.core.log.info(_("Download aborted: %s") % pyfile.name) - pyfile.setStatus("aborted") - - self.clean(pyfile) - else: - self.queue.put(pyfile) - - continue - - else: - pyfile.setStatus("failed") - self.m.core.log.error("pycurl error %s: %s" % (code, msg)) - if self.m.core.debug: - traceback.print_exc() - self.writeDebugReport(pyfile) - - self.m.core.addonManager.downloadFailed(pyfile) - - self.clean(pyfile) - continue - - except SkipDownload, e: - pyfile.setStatus("skipped") - - self.m.core.log.info(_("Download skipped: %(name)s due to %(plugin)s") % {"name": pyfile.name, "plugin": e.message}) - - self.clean(pyfile) - - self.m.core.files.checkPackageFinished(pyfile) - - self.active = False - self.m.core.files.save() - - continue - - except Exception, e: - pyfile.setStatus("failed") - self.m.core.log.warning(_("Download failed: %(name)s | %(msg)s") % {"name": pyfile.name, "msg": str(e)}) - pyfile.error = str(e) - - if self.m.core.debug: - traceback.print_exc() - self.writeDebugReport(pyfile) - - self.m.core.addonManager.downloadFailed(pyfile) - self.clean(pyfile) - continue - - finally: - self.m.core.files.save() - pyfile.checkIfProcessed() - exc_clear() - - # pyfile.plugin.req.clean() - - self.active = False - pyfile.finishIfDone() - self.m.core.files.save() - - - def put(self, job): - """assing job to thread""" - self.queue.put(job) - - - def stop(self): - """stops the thread""" - self.put("quit") diff --git a/pyload/manager/thread/Info.py b/pyload/manager/thread/Info.py deleted file mode 100644 index 856c8facf..000000000 --- a/pyload/manager/thread/Info.py +++ /dev/null @@ -1,221 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -import traceback - -from Queue import Queue -from copy import copy -from os import listdir, stat -from os.path import join -from pprint import pformat -from sys import exc_info, exc_clear -from time import sleep, time, strftime, gmtime -from types import MethodType - -from pyload.api import OnlineStatus -from pyload.datatype.File import PyFile -from pyload.manager.thread.Plugin import PluginThread - - -class InfoThread(PluginThread): - - def __init__(self, manager, data, pid=-1, rid=-1, add=False): - """Constructor""" - PluginThread.__init__(self, manager) - - self.data = data - self.pid = pid #: package id - # [ .. (name, plugin) .. ] - - self.rid = rid #: result id - self.add = add #: add packages instead of return result - - self.cache = [] #: accumulated data - - self.start() - - - def run(self): - """run method""" - - plugins = {} - container = [] - - for url, plugintype, pluginname in self.data: - # filter out container plugins - if plugintype == 'container': - container.appen((pluginname, url)) - else: - if (plugintype, pluginname) in plugins: - plugins[(plugintype, pluginname)].append(url) - else: - plugins[(plugintype, pluginname)] = [url] - - # directly write to database - if self.pid > -1: - for (plugintype, pluginname), urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateDB) - self.m.core.files.save() - - elif self.add: - for (plugintype, pluginname), urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateCache, True) - - else: - # generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateCache(pluginname, result) - - packs = parseNames([(name, url) for name, x, y, url in self.cache]) - - self.m.log.debug("Fetched and generated %d packages" % len(packs)) - - for k, v in packs: - self.m.core.api.addPackage(k, v) - - # empty cache - del self.cache[:] - - else: #: post the results - - for name, url in container: - # attach container content - try: - data = self.decryptContainer(name, url) - except Exception: - traceback.print_exc() - self.m.log.error("Could not decrypt container.") - data = [] - - for url, plugintype, pluginname in data: - try: - plugins[plugintype][pluginname].append(url) - except Exception: - plugins[plugintype][pluginname] = [url] - - self.m.infoResults[self.rid] = {} - - for plugintype, pluginname, urls in plugins.iteritems(): - plugin = self.m.core.pluginManager.getPlugin(plugintype, pluginname, True) - if hasattr(plugin, "getInfo"): - self.fetchForPlugin(pluginname, plugin, urls, self.updateResult, True) - - # force to process cache - if self.cache: - self.updateResult(pluginname, [], True) - - else: - # generate default result - result = [(url, 0, 3, url) for url in urls] - - self.updateResult(pluginname, result, True) - - self.m.infoResults[self.rid]['ALL_INFO_FETCHED'] = {} - - self.m.timestamp = time() + 5 * 60 - - - def updateDB(self, plugin, result): - self.m.core.files.updateFileInfo(result, self.pid) - - - def updateResult(self, plugin, result, force=False): - # parse package name and generate result - # accumulate results - - self.cache.extend(result) - - if len(self.cache) >= 20 or force: - # used for package generating - tmp = [(name, (url, OnlineStatus(name, plugin, "unknown", status, int(size)))) for name, size, status, url in self.cache] - - data = parseNames(tmp) - result = {} - for k, v in data.iteritems(): - for url, status in v: - status.packagename = k - result[url] = status - - self.m.setInfoResults(self.rid, result) - - self.cache = [] - - - def updateCache(self, plugin, result): - self.cache.extend(result) - - - def fetchForPlugin(self, pluginname, plugin, urls, cb, err=None): - try: - result = [] #: result loaded from cache - process = [] #: urls to process - for url in urls: - if url in self.m.infoCache: - result.append(self.m.infoCache[url]) - else: - process.append(url) - - if result: - self.m.log.debug("Fetched %d values from cache for %s" % (len(result), pluginname)) - cb(pluginname, result) - - if process: - self.m.log.debug("Run Info Fetching for %s" % pluginname) - for result in plugin.getInfo(process): - # result = [ .. (name, size, status, url) .. ] - if not type(result) == list: - result = [result] - - for res in result: - self.m.infoCache[res[3]] = res # : why don't assign res dict directly? - - cb(pluginname, result) - - self.m.log.debug("Finished Info Fetching for %s" % pluginname) - except Exception, e: - self.m.log.warning(_("Info Fetching for %(name)s failed | %(err)s") % {"name": pluginname, "err": str(e)}) - if self.m.core.debug: - traceback.print_exc() - - # generate default results - if err: - result = [(url, 0, 3, url) for url in urls] - cb(pluginname, result) - - - def decryptContainer(self, plugin, url): - data = [] - # only works on container plugins - - self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) - - # dummy pyfile - pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) - - pyfile.initPlugin() - - # little plugin lifecycle - try: - pyfile.plugin.setup() - pyfile.plugin.loadToDisk() - pyfile.plugin.decrypt(pyfile) - pyfile.plugin.deleteTmp() - - for pack in pyfile.plugin.packages: - pyfile.plugin.urls.extend(pack[1]) - - data = self.m.core.pluginManager.parseUrls(pyfile.plugin.urls) - - self.m.log.debug("Got %d links." % len(data)) - - except Exception, e: - self.m.log.debug("Pre decrypting error: %s" % str(e)) - finally: - pyfile.release() - - return data diff --git a/pyload/manager/thread/Plugin.py b/pyload/manager/thread/Plugin.py deleted file mode 100644 index 0163152f3..000000000 --- a/pyload/manager/thread/Plugin.py +++ /dev/null @@ -1,132 +0,0 @@ -# -*- coding: utf-8 -*- -# @author: RaNaN - -from __future__ import with_statement - -import threading -import traceback - -from Queue import Queue -from copy import copy -from os import listdir, stat -from os.path import join -from pprint import pformat -from sys import exc_info, exc_clear -from time import sleep, time, strftime, gmtime -from types import MethodType - -from pyload.api import OnlineStatus -from pyload.datatype.File import PyFile -from pyload.plugin.Plugin import Abort, Fail, Reconnect, Retry, SkipDownload -from pyload.utils.packagetools import parseNames -from pyload.utils import fs_join - - -class PluginThread(threading.Thread): - """abstract base class for thread types""" - - def __init__(self, manager): - """Constructor""" - threading.Thread.__init__(self) - self.setDaemon(True) - self.m = manager #: thread manager - - - def writeDebugReport(self, pyfile): - """ writes a - :return: - """ - - dump_name = "debug_%s_%s.zip" % (pyfile.pluginname, strftime("%d-%m-%Y_%H-%M-%S")) - dump = self.getDebugDump(pyfile) - - try: - import zipfile - - zip = zipfile.ZipFile(dump_name, "w") - - for f in listdir(join("tmp", pyfile.pluginname)): - try: - # avoid encoding errors - zip.write(join("tmp", pyfile.pluginname, f), fs_join(pyfile.pluginname, f)) - except Exception: - pass - - info = zipfile.ZipInfo(fs_join(pyfile.pluginname, "debug_Report.txt"), gmtime()) - info.external_attr = 0644 << 16L #: change permissions - - zip.writestr(info, dump) - zip.close() - - if not stat(dump_name).st_size: - raise Exception("Empty Zipfile") - - except Exception, e: - self.m.log.debug("Error creating zip file: %s" % e) - - dump_name = dump_name.replace(".zip", ".txt") - with open(dump_name, "wb") as f: - f.write(dump) - - self.m.core.log.info("Debug Report written to %s" % dump_name) - - - def getDebugDump(self, pyfile): - dump = "pyLoad %s Debug Report of %s %s \n\nTRACEBACK:\n %s \n\nFRAMESTACK:\n" % ( - self.m.core.api.getServerVersion(), pyfile.pluginname, pyfile.plugin.__version, traceback.format_exc()) - - tb = exc_info()[2] - stack = [] - while tb: - stack.append(tb.tb_frame) - tb = tb.tb_next - - for frame in stack[1:]: - dump += "\nFrame %s in %s at line %s\n" % (frame.f_code.co_name, - frame.f_code.co_filename, - frame.f_lineno) - - for key, value in frame.f_locals.items(): - dump += "\t%20s = " % key - try: - dump += pformat(value) + "\n" - except Exception, e: - dump += " " + str(e) + "\n" - - del frame - - del stack #: delete it just to be sure... - - dump += "\n\nPLUGIN OBJECT DUMP: \n\n" - - for name in dir(pyfile.plugin): - attr = getattr(pyfile.plugin, name) - if not name.endswith("__") and type(attr) != MethodType: - dump += "\t%20s = " % name - try: - dump += pformat(attr) + "\n" - except Exception, e: - dump += " " + str(e) + "\n" - - dump += "\nPYFILE OBJECT DUMP: \n\n" - - for name in dir(pyfile): - attr = getattr(pyfile, name) - if not name.endswith("__") and type(attr) != MethodType: - dump += "\t%20s = " % name - try: - dump += pformat(attr) + "\n" - except Exception, e: - dump += " " + str(e) + "\n" - - if pyfile.pluginname in self.m.core.config.plugin: - dump += "\n\nCONFIG: \n\n" - dump += pformat(self.m.core.config.plugin[pyfile.pluginname]) + "\n" - - return dump - - - def clean(self, pyfile): - """ set thread unactive and release pyfile """ - self.active = True #: release pyfile but lets the thread active - pyfile.release() diff --git a/pyload/manager/thread/Server.py b/pyload/manager/thread/Server.py deleted file mode 100644 index a26ffe6a1..000000000 --- a/pyload/manager/thread/Server.py +++ /dev/null @@ -1,122 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import with_statement - -import logging -import os -import threading - -core = None -setup = None -log = logging.getLogger("log") - - -class WebServer(threading.Thread): - - def __init__(self, pycore): - global core - - threading.Thread.__init__(self) - self.core = pycore - core = pycore - self.running = True - self.server = pycore.config.get("webui", "server") - self.https = pycore.config.get("webui", "https") - self.cert = pycore.config.get("ssl", "cert") - self.key = pycore.config.get("ssl", "key") - self.host = pycore.config.get("webui", "host") - self.port = pycore.config.get("webui", "port") - - self.setDaemon(True) - - - def run(self): - import pyload.webui as webinterface - global webinterface - - reset = False - - if self.https and (not os.exists(self.cert) or not os.exists(self.key)): - log.warning(_("SSL certificates not found.")) - self.https = False - - if self.server in ("lighttpd", "nginx"): - log.warning(_("Sorry, we dropped support for starting %s directly within pyLoad") % self.server) - log.warning(_("You can use the threaded server which offers good performance and ssl,")) - log.warning(_("of course you can still use your existing %s with pyLoads fastcgi server") % self.server) - log.warning(_("sample configs are located in the pyload/web/servers directory")) - reset = True - elif self.server == "fastcgi": - try: - import flup - except Exception: - log.warning(_("Can't use %(server)s, python-flup is not installed!") % { - "server": self.server}) - reset = True - - if reset or self.server == "lightweight": - if os.name != "nt": - try: - import bjoern - except Exception, e: - log.error(_("Error importing lightweight server: %s") % e) - log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern")) - log.warning(_("Copy the boern.so file to lib/Python/Lib or use setup.py install")) - log.warning(_("Of course you need to be familiar with linux and know how to compile software")) - self.server = "auto" - else: - self.core.log.info(_("Server set to threaded, due to known performance problems on windows.")) - self.core.config.set("webui", "server", "threaded") - self.server = "threaded" - - if self.server == "threaded": - self.start_threaded() - elif self.server == "fastcgi": - self.start_fcgi() - elif self.server == "lightweight": - self.start_lightweight() - else: - self.start_auto() - - - def start_auto(self): - if self.https: - log.warning(_("This server offers no SSL, please consider using `threaded` instead")) - - self.core.log.info(_("Starting builtin webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_auto(host=self.host, port=self.port) - - - def start_threaded(self): - if self.https: - self.core.log.info( - _("Starting threaded SSL webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - else: - self.cert = "" - self.key = "" - self.core.log.info( - _("Starting threaded webserver: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - - webinterface.run_threaded(host=self.host, port=self.port, cert=self.cert, key=self.key) - - - def start_fcgi(self): - self.core.log.info(_("Starting fastcgi server: %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - try: - webinterface.run_fcgi(host=self.host, port=self.port) - - except ValueError: #@TODO: Fix https://github.com/pyload/pyload/issues/1145 - pass - - - def start_lightweight(self): - if self.https: - log.warning(_("This server offers no SSL, please consider using `threaded` instead")) - - self.core.log.info( - _("Starting lightweight webserver (bjoern): %(host)s:%(port)d") % {"host": self.host, "port": self.port}) - webinterface.run_lightweight(host=self.host, port=self.port) - - - def quit(self): - self.running = False diff --git a/pyload/manager/thread/__init__.py b/pyload/manager/thread/__init__.py deleted file mode 100644 index 40a96afc6..000000000 --- a/pyload/manager/thread/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- diff --git a/pyload/plugin/Extractor.py b/pyload/plugin/Extractor.py index 539ab624d..3e497fc78 100644 --- a/pyload/plugin/Extractor.py +++ b/pyload/plugin/Extractor.py @@ -3,7 +3,7 @@ import os import re -from pyload.datatype.File import PyFile +from pyload.Datatype import PyFile from pyload.plugin.Plugin import Base diff --git a/pyload/webui/__init__.py b/pyload/webui/__init__.py index 3b3d62b26..665be763a 100644 --- a/pyload/webui/__init__.py +++ b/pyload/webui/__init__.py @@ -24,7 +24,7 @@ from middlewares import StripPathMiddleware, GZipMiddleWare, PrefixMiddleware SETUP = None PYLOAD = None -from pyload.manager.thread import Server +from pyload.Thread import Server from pyload.network.JsEngine import JsEngine if not Server.core: -- cgit v1.2.3