diff options
Diffstat (limited to 'module')
| -rw-r--r-- | module/Api.py | 53 | ||||
| -rw-r--r-- | module/database/UserDatabase.py | 5 | ||||
| -rw-r--r-- | module/remote/thriftbackend/ThriftClient.py | 4 | ||||
| -rwxr-xr-x | module/remote/thriftbackend/generateThrift.sh | 4 | ||||
| -rw-r--r-- | module/remote/thriftbackend/pyload.thrift | 2 | ||||
| -rwxr-xr-x | module/remote/thriftbackend/thriftgen/pyload/Pyload-remote | 8 | ||||
| -rw-r--r-- | module/remote/thriftbackend/thriftgen/pyload/Pyload.py | 21 | ||||
| -rw-r--r-- | module/web/api_app.py | 107 | ||||
| -rw-r--r-- | module/web/pyload_app.py | 12 | ||||
| -rw-r--r-- | module/web/utils.py | 20 | ||||
| -rw-r--r-- | module/web/webinterface.py | 1 |
11 files changed, 189 insertions, 48 deletions
diff --git a/module/Api.py b/module/Api.py index 54aeff669..5583bb603 100644 --- a/module/Api.py +++ b/module/Api.py @@ -20,6 +20,7 @@ from base64 import standard_b64encode from os.path import join from time import time +import re from remote.thriftbackend.thriftgen.pyload.ttypes import * from remote.thriftbackend.thriftgen.pyload.Pyload import Iface @@ -28,6 +29,11 @@ from PyFile import PyFile from database.UserDatabase import ROLE from utils import freeSpace, compare_time from common.packagetools import parseNames +from network.RequestFactory import getURL + + +urlmatcher = re.compile(r"((https?|ftps?|xdcc|sftp):((//)|(\\\\))+[\w\d:#@%/;$()~_?\+-=\\\.&]*)", re.IGNORECASE) + class Api(Iface): """ @@ -39,6 +45,8 @@ class Api(Iface): for information about data structures and what methods are usuable with rpc. """ + EXTERNAL = Iface # let the json api know which methods are external + def __init__(self, core): self.core = core @@ -54,7 +62,7 @@ class Api(Iface): section = ConfigSection(sectionName, sub["desc"]) items = [] for key, data in sub.iteritems(): - if key in ("desc","outline"): + if key in ("desc", "outline"): continue item = ConfigItem() item.name = key @@ -130,7 +138,7 @@ class Api(Iface): """ return self.core.config.plugin - + def pauseServer(self): """Pause server: Tt wont start any new downloads, but nothing gets aborted.""" self.core.threadManager.pause = True @@ -175,7 +183,7 @@ class Api(Iface): return freeSpace(self.core.config["general"]["download_folder"]) def getServerVersion(self): - """pyLoad Core version """ + """pyLoad Core version """ return self.core.version def kill(self): @@ -266,14 +274,30 @@ class Api(Iface): return pid - def parseURLs(self, html): - # TODO parse + def parseURLs(self, html=None, url=None): + """Parses html content or any arbitaty text for links and returns result of `checkURLs` + + :param html: html source + :return: + """ urls = [] + if html: + urls += [x[0] for x in urlmatcher.findall(html)] + + if url: + page = getURL(url) + urls += [x[0] for x in urlmatcher.findall(page)] + return self.checkURLs(urls) def checkURLs(self, urls): + """ Gets urls and returns pluginname mapped to list of matches urls. + + :param urls: + :return: {plugin: urls} + """ data = self.core.pluginManager.parseUrls(urls) plugins = {} @@ -289,7 +313,7 @@ class Api(Iface): """ initiates online status check :param urls: - :return: initial set of data and the result id + :return: initial set of data as `OnlineCheck` instance containing the result id """ data = self.core.pluginManager.parseUrls(urls) @@ -298,7 +322,7 @@ class Api(Iface): tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data] data = parseNames(tmp) result = {} - + for k, v in data.iteritems(): for url, status in v: status.packagename = k @@ -307,7 +331,7 @@ class Api(Iface): return OnlineCheck(rid, result) def checkOnlineStatusContainer(self, urls, container, data): - """ checks online status of files and container file + """ checks online status of urls and a submited container file :param urls: list of urls :param container: container file name @@ -323,8 +347,8 @@ class Api(Iface): def pollResults(self, rid): """ Polls the result available for ResultID - :param rid: if -1 no more data is available - :return: + :param rid: `ResultID` + :return: `OnlineCheck`, if rid is -1 then no more data available """ result = self.core.threadManager.getInfoResult(rid) @@ -336,12 +360,12 @@ class Api(Iface): def generatePackages(self, links): - """ Parses links, generates packages names only from urls + """ Parses links, generates packages names from urls :param links: list of urls - :return: package names mapt to urls + :return: package names mapped to urls """ - result = parseNames((x,x) for x in links) + result = parseNames((x, x) for x in links) return result def generateAndAddPackages(self, links, dest=Destination.Queue): @@ -365,7 +389,6 @@ class Api(Iface): data = self.core.pluginManager.parseUrls(links) self.core.threadManager.createResultThread(data, True) - def getPackageData(self, pid): """Returns complete information about package, and included files. @@ -410,7 +433,7 @@ class Api(Iface): info = self.core.files.getFileData(int(fid)) if not info: raise FileDoesNotExists(fid) - + fdata = self._convertPyFile(info.values()[0]) return fdata diff --git a/module/database/UserDatabase.py b/module/database/UserDatabase.py index 0e3011593..f888e219e 100644 --- a/module/database/UserDatabase.py +++ b/module/database/UserDatabase.py @@ -23,13 +23,14 @@ from hashlib import sha1 import random class PERMS: + ALL = 0 # requires no permission, but login ADD = 1 # can add packages DELETE = 2 # can delete packages STATUS = 4 # see and change server status - SEE_DOWNLOADS = 16 # see queue and collector + SEE_DOWNLOADS = 16 # see queue and collector / modify downloads DOWNLOAD = 32 # can download from webinterface SETTINGS = 64 # can access settings - FILEMANAGER = 128 # can manage files and folders trough webinterface + ACCOUNTS = 128 # can access accounts class ROLE: ADMIN = 0 #admin has all permissions implicit diff --git a/module/remote/thriftbackend/ThriftClient.py b/module/remote/thriftbackend/ThriftClient.py index 3b6a56448..27cb89b0d 100644 --- a/module/remote/thriftbackend/ThriftClient.py +++ b/module/remote/thriftbackend/ThriftClient.py @@ -17,9 +17,7 @@ from Protocol import Protocol from thriftgen.pyload import Pyload from thriftgen.pyload.ttypes import * -from thriftgen.pyload.Pyload import PackageDoesNotExists -from thriftgen.pyload.Pyload import FileDoesNotExists - +from thriftgen.pyload.Pyload import PackageDoesNotExists, FileDoesNotExists ConnectionClosed = TTransport.TTransportException diff --git a/module/remote/thriftbackend/generateThrift.sh b/module/remote/thriftbackend/generateThrift.sh index 7acb8531c..c5ca72736 100755 --- a/module/remote/thriftbackend/generateThrift.sh +++ b/module/remote/thriftbackend/generateThrift.sh @@ -1,9 +1,7 @@ #!/bin/sh rm -rf thriftgen -# use thrift 0.7.0 from trunk -# patched if needed https://issues.apache.org/jira/browse/THRIFT-1115?page=com.atlassian.jira.plugin.system.issuetabpanels%3Achangehistory-tabpanel#issue-tabs -# --gen py:slots,dynamic +# use thrift from trunk or a release with dynamic/slots python code generation thrift -strict -v --gen py --gen java pyload.thrift mv gen-py thriftgen diff --git a/module/remote/thriftbackend/pyload.thrift b/module/remote/thriftbackend/pyload.thrift index e7ae10055..dc085e180 100644 --- a/module/remote/thriftbackend/pyload.thrift +++ b/module/remote/thriftbackend/pyload.thrift @@ -211,7 +211,7 @@ service Pyload { // packagename - urls map<string, LinkList> generatePackages(1: LinkList links), map<PluginName, LinkList> checkURLs(1: LinkList urls), - map<PluginName, LinkList> parseURLs(1: string html), + map<PluginName, LinkList> parseURLs(1: string html, 2: string url), // parses results and generates packages OnlineCheck checkOnlineStatus(1: LinkList urls), diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote index 6c8116d92..0251fbeae 100755 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload-remote @@ -41,7 +41,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print ' bool toggleReconnect()' print ' generatePackages(LinkList links)' print ' checkURLs(LinkList urls)' - print ' parseURLs(string html)' + print ' parseURLs(string html, string url)' print ' OnlineCheck checkOnlineStatus(LinkList urls)' print ' OnlineCheck checkOnlineStatusContainer(LinkList urls, string filename, string data)' print ' OnlineCheck pollResults(ResultID rid)' @@ -251,10 +251,10 @@ elif cmd == 'checkURLs': pp.pprint(client.checkURLs(eval(args[0]),)) elif cmd == 'parseURLs': - if len(args) != 1: - print 'parseURLs requires 1 args' + if len(args) != 2: + print 'parseURLs requires 2 args' sys.exit(1) - pp.pprint(client.parseURLs(args[0],)) + pp.pprint(client.parseURLs(args[0],args[1],)) elif cmd == 'checkOnlineStatus': if len(args) != 1: diff --git a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py index a29eed62f..828ff520d 100644 --- a/module/remote/thriftbackend/thriftgen/pyload/Pyload.py +++ b/module/remote/thriftbackend/thriftgen/pyload/Pyload.py @@ -9,7 +9,7 @@ from thrift.Thrift import TType, TMessageType from ttypes import * from thrift.Thrift import TProcessor -from thrift.protocol.TBase import TBase, TExceptionBase +from thrift.protocol.TBase import TBase, TExceptionBase, TApplicationException class Iface(object): @@ -92,10 +92,11 @@ class Iface(object): """ pass - def parseURLs(self, html): + def parseURLs(self, html, url): """ Parameters: - html + - url """ pass @@ -905,18 +906,20 @@ class Client(Iface): return result.success raise TApplicationException(TApplicationException.MISSING_RESULT, "checkURLs failed: unknown result"); - def parseURLs(self, html): + def parseURLs(self, html, url): """ Parameters: - html + - url """ - self.send_parseURLs(html) + self.send_parseURLs(html, url) return self.recv_parseURLs() - def send_parseURLs(self, html): + def send_parseURLs(self, html, url): self._oprot.writeMessageBegin('parseURLs', TMessageType.CALL, self._seqid) args = parseURLs_args() args.html = html + args.url = url args.write(self._oprot) self._oprot.writeMessageEnd() self._oprot.trans.flush() @@ -2678,7 +2681,7 @@ class Processor(Iface, TProcessor): args.read(iprot) iprot.readMessageEnd() result = parseURLs_result() - result.success = self._handler.parseURLs(args.html) + result.success = self._handler.parseURLs(args.html, args.url) oprot.writeMessageBegin("parseURLs", TMessageType.REPLY, seqid) result.write(oprot) oprot.writeMessageEnd() @@ -3769,19 +3772,23 @@ class parseURLs_args(TBase): """ Attributes: - html + - url """ __slots__ = [ 'html', + 'url', ] thrift_spec = ( None, # 0 (1, TType.STRING, 'html', None, None, ), # 1 + (2, TType.STRING, 'url', None, None, ), # 2 ) - def __init__(self, html=None,): + def __init__(self, html=None, url=None,): self.html = html + self.url = url class parseURLs_result(TBase): diff --git a/module/web/api_app.py b/module/web/api_app.py new file mode 100644 index 000000000..7d1a95c58 --- /dev/null +++ b/module/web/api_app.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from urllib import unquote +from itertools import chain +from traceback import format_exc, print_exc + +from bottle import route, request, response, HTTPError + +from thrift.protocol.TBase import TBase + +from utils import toDict, set_session + +from webinterface import PYLOAD + +from module.common.json import json_dumps +from module.database.UserDatabase import ROLE + +try: + from ast import literal_eval +except ImportError: # python 2.5 + from module.lib.SafeEval import safe_eval as literal_eval + + +# convert to format serializable by json +def traverse(data): + if type(data) == list: + return [traverse(x) for x in data] + elif type(data) == dict: + return dict([(x, traverse(y)) for x, y in data.iteritems()]) + elif isinstance(data, TBase): + return toDict(data) + else: + return data + + +# accepting positional arguments, as well as kwargs via post and get + +@route("/api/:func:args#[a-zA-Z0-9\-_/\"'\[\]%{}]*#") +@route("/api/:func:args#[a-zA-Z0-9\-_/\"'\[\]%{}]*#", method="POST") +def call_api(func, args=""): + response.headers.replace("Content-type", "application/json") + response.headers.append("Cache-Control", "no-cache, must-revalidate") + + s = request.environ.get('beaker.session') + if 'session' in request.POST: + s = s.get_by_id(request.POST['session']) + + if not s or not s.get("authenticated", False) or s.get("role", -1) != ROLE.ADMIN: + return HTTPError(401, json_dumps("Unauthorized")) + + args = args.split("/")[1:] + kwargs = {} + + for x, y in chain(request.GET.iteritems(), request.POST.iteritems()): + if x == "session": continue + kwargs[x] = unquote(y) + + try: + return callApi(func, *args, **kwargs) + except Exception, e: + print_exc() + return HTTPError(500, json_dumps({"error": e.message, "traceback": format_exc()})) + + +def callApi(func, *args, **kwargs): + if not hasattr(PYLOAD.EXTERNAL, func) or func.startswith("_"): + print "Invalid API call", func + return HTTPError(404, json_dumps("Not Found")) + + result = getattr(PYLOAD, func)(*[literal_eval(x) for x in args], + **dict([(x, literal_eval(y)) for x, y in kwargs.iteritems()])) + + return json_dumps(traverse(result)) + + +#post -> username, password +@route("/api/login", method="POST") +def login(): + response.headers.replace("Content-type", "application/json") + response.headers.append("Cache-Control", "no-cache, must-revalidate") + + user = request.forms.get("username") + password = request.forms.get("password") + + info = PYLOAD.checkAuth(user, password) + + if not info: + return json_dumps(False) + + s = set_session(request, info) + + # get the session id by dirty way, documentations seems wrong + try: + sid = s._headers["cookie_out"].split("=")[1].split(";")[0] + return json_dumps(sid) + except: + return json_dumps(True) + + +@route("/api/logout") +def logout(): + response.headers.replace("Content-type", "application/json") + response.headers.append("Cache-Control", "no-cache, must-revalidate") + + s = request.environ.get('beaker.session') + s.delete() diff --git a/module/web/pyload_app.py b/module/web/pyload_app.py index 8d76d39ec..49568baad 100644 --- a/module/web/pyload_app.py +++ b/module/web/pyload_app.py @@ -32,7 +32,7 @@ from bottle import route, static_file, request, response, redirect, HTTPError, e from webinterface import PYLOAD, PYLOAD_DIR, PROJECT_DIR, SETUP from utils import render_to_response, parse_permissions, parse_userdata, \ - login_required, get_permission, set_permission, toDict + login_required, get_permission, set_permission, toDict, set_session from filters import relpath, unquotepath @@ -119,15 +119,7 @@ def login_post(): if not info: return render_to_response("login.html", {"errors": True}, [pre_processor]) - s = request.environ.get('beaker.session') - s["authenticated"] = True - s["id"] = info["id"] - s["name"] = info["name"] - s["role"] = info["role"] - s["perms"] = info["permission"] - s["template"] = info["template"] - s.save() - + set_session(request, info) return redirect("/") diff --git a/module/web/utils.py b/module/web/utils.py index b99736216..39ddb361f 100644 --- a/module/web/utils.py +++ b/module/web/utils.py @@ -52,6 +52,7 @@ def parse_permissions(session): return perms + def get_permission(perms, p): perms["add"] = has_permission(p, PERMS.ADD) perms["delete"] = has_permission(p, PERMS.DELETE) @@ -59,7 +60,7 @@ def get_permission(perms, p): perms["see_downloads"] = has_permission(p, PERMS.SEE_DOWNLOADS) perms["download"] = has_permission(p, PERMS.DOWNLOAD) perms["settings"] = has_permission(p, PERMS.SETTINGS) - perms["filemanager"] = has_permission(p, PERMS.FILEMANAGER) + perms["accounts"] = has_permission(p, PERMS.ACCOUNTS) def set_permission(perms): permission = 0 @@ -75,11 +76,24 @@ def set_permission(perms): permission |= PERMS.DOWNLOAD if perms["settings"]: permission |= PERMS.SETTINGS - if perms["filemanager"]: - permission |= PERMS.FILEMANAGER + if perms["accounts"]: + permission |= PERMS.ACCOUNTS return permission + +def set_session(request, info): + s = request.environ.get('beaker.session') + s["authenticated"] = True + s["user_id"] = info["id"] + s["name"] = info["name"] + s["role"] = info["role"] + s["perms"] = info["permission"] + s["template"] = info["template"] + s.save() + + return s + def parse_userdata(session): return {"name": session.get("name", "Anonymous"), "is_admin": True if session.get("role", 1) == 0 else False, diff --git a/module/web/webinterface.py b/module/web/webinterface.py index 4d07c436e..d2b96b03c 100644 --- a/module/web/webinterface.py +++ b/module/web/webinterface.py @@ -105,6 +105,7 @@ web = GZipMiddleWare(web) import pyload_app import json_app import cnl_app +import api_app def run_simple(host="0.0.0.0", port="8000"): run(app=web, host=host, port=port, quiet=True) |