diff options
Diffstat (limited to 'module/common')
| -rw-r--r-- | module/common/APIExerciser.py | 157 | ||||
| -rw-r--r-- | module/common/ImportDebugger.py | 19 | ||||
| -rw-r--r-- | module/common/JsEngine.py | 162 | ||||
| -rw-r--r-- | module/common/__init__.py | 2 | ||||
| -rw-r--r-- | module/common/json_layer.py | 13 | ||||
| -rw-r--r-- | module/common/packagetools.py | 155 | ||||
| -rw-r--r-- | module/common/pylgettext.py | 61 |
7 files changed, 0 insertions, 569 deletions
diff --git a/module/common/APIExerciser.py b/module/common/APIExerciser.py deleted file mode 100644 index 96f5ce9cf..000000000 --- a/module/common/APIExerciser.py +++ /dev/null @@ -1,157 +0,0 @@ -# -*- coding: utf-8 -*- - -import string -from threading import Thread -from random import choice, random, sample, randint -from time import time, sleep -from math import floor -import gc - -from traceback import print_exc, format_exc - -from module.remote.thriftbackend.ThriftClient import ThriftClient, Destination - -def createURLs(): - """ create some urls, some may fail """ - urls = [] - for x in range(0, randint(20, 100)): - name = "DEBUG_API" - if randint(0, 5) == 5: - name = "" #this link will fail - - urls.append(name + "".join(sample(string.ascii_letters, randint(10, 20)))) - - return urls - -AVOID = (0,3,8) - -idPool = 0 -sumCalled = 0 - - -def startApiExerciser(core, n): - for i in range(n): - APIExerciser(core).start() - -class APIExerciser(Thread): - - - def __init__(self, core, thrift=False, user=None, pw=None): - global idPool - - Thread.__init__(self) - self.setDaemon(True) - self.core = core - self.count = 0 #number of methods - self.time = time() - - if thrift: - self.api = ThriftClient(user=user, password=pw) - else: - self.api = core.api - - - self.id = idPool - - idPool += 1 - - #self.start() - - def run(self): - - self.core.log.info("API Excerciser started %d" % self.id) - - out = open("error.log", "ab") - #core errors are not logged of course - out.write("\n" + "Starting\n") - out.flush() - - while True: - try: - self.testAPI() - except Exception: - self.core.log.error("Excerciser %d throw an execption" % self.id) - print_exc() - out.write(format_exc() + 2 * "\n") - out.flush() - - if not self.count % 100: - self.core.log.info("Exerciser %d tested %d api calls" % (self.id, self.count)) - if not self.count % 1000: - out.flush() - - if not sumCalled % 1000: #not thread safe - self.core.log.info("Exercisers tested %d api calls" % sumCalled) - persec = sumCalled / (time() - self.time) - self.core.log.info("Approx. %.2f calls per second." % persec) - self.core.log.info("Approx. %.2f ms per call." % (1000 / persec)) - self.core.log.info("Collected garbage: %d" % gc.collect()) - - - #sleep(random() / 500) - - def testAPI(self): - global sumCalled - - m = ["statusDownloads", "statusServer", "addPackage", "getPackageData", "getFileData", "deleteFiles", - "deletePackages", "getQueue", "getCollector", "getQueueData", "getCollectorData", "isCaptchaWaiting", - "getCaptchaTask", "stopAllDownloads", "getAllInfo", "getServices" , "getAccounts", "getAllUserData"] - - method = choice(m) - #print "Testing:", method - - if hasattr(self, method): - res = getattr(self, method)() - else: - res = getattr(self.api, method)() - - self.count += 1 - sumCalled += 1 - - #print res - - def addPackage(self): - name = "".join(sample(string.ascii_letters, 10)) - urls = createURLs() - - self.api.addPackage(name, urls, choice([Destination.Queue, Destination.Collector])) - - - def deleteFiles(self): - info = self.api.getQueueData() - if not info: return - - pack = choice(info) - fids = pack.links - - if len(fids): - fids = [f.fid for f in sample(fids, randint(1, max(len(fids) / 2, 1)))] - self.api.deleteFiles(fids) - - - def deletePackages(self): - info = choice([self.api.getQueue(), self.api.getCollector()]) - if not info: return - - pids = [p.pid for p in info] - if len(pids): - pids = sample(pids, randint(1, max(floor(len(pids) / 2.5), 1))) - self.api.deletePackages(pids) - - def getFileData(self): - info = self.api.getQueueData() - if info: - p = choice(info) - if p.links: - self.api.getFileData(choice(p.links).fid) - - def getPackageData(self): - info = self.api.getQueue() - if info: - self.api.getPackageData(choice(info).pid) - - def getAccounts(self): - self.api.getAccounts(False) - - def getCaptchaTask(self): - self.api.getCaptchaTask(False)
\ No newline at end of file diff --git a/module/common/ImportDebugger.py b/module/common/ImportDebugger.py deleted file mode 100644 index a997f7b0c..000000000 --- a/module/common/ImportDebugger.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys - -class ImportDebugger(object): - - def __init__(self): - self.imported = {} - - def find_module(self, name, path=None): - - if name not in self.imported: - self.imported[name] = 0 - - self.imported[name] += 1 - - print name, path - -sys.meta_path.append(ImportDebugger())
\ No newline at end of file diff --git a/module/common/JsEngine.py b/module/common/JsEngine.py deleted file mode 100644 index 576be2a1b..000000000 --- a/module/common/JsEngine.py +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 3 of the License, - or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, see <http://www.gnu.org/licenses/>. - - @author: RaNaN -""" - -from imp import find_module -from os.path import join, exists -from urllib import quote - - -ENGINE = "" - -DEBUG = False -JS = False -PYV8 = False -RHINO = False - - -if not ENGINE: - try: - import subprocess - - subprocess.Popen(["js", "-v"], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - p = subprocess.Popen(["js", "-e", "print(23+19)"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - #integrity check - if out.strip() == "42": - ENGINE = "js" - JS = True - except: - pass - -if not ENGINE or DEBUG: - try: - find_module("PyV8") - ENGINE = "pyv8" - PYV8 = True - except: - pass - -if not ENGINE or DEBUG: - try: - path = "" #path where to find rhino - - if exists("/usr/share/java/js.jar"): - path = "/usr/share/java/js.jar" - elif exists("js.jar"): - path = "js.jar" - elif exists(join(pypath, "js.jar")): #may raises an exception, but js.jar wasnt found anyway - path = join(pypath, "js.jar") - - if not path: - raise Exception - - import subprocess - - p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", "print(23+19)"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - #integrity check - if out.strip() == "42": - ENGINE = "rhino" - RHINO = True - except: - pass - -class JsEngine(): - def __init__(self): - self.engine = ENGINE - self.init = False - - def __nonzero__(self): - return False if not ENGINE else True - - def eval(self, script): - if not self.init: - if ENGINE == "pyv8" or (DEBUG and PYV8): - import PyV8 - global PyV8 - - self.init = True - - if type(script) == unicode: - script = script.encode("utf8") - - if not ENGINE: - raise Exception("No JS Engine") - - if not DEBUG: - if ENGINE == "pyv8": - return self.eval_pyv8(script) - elif ENGINE == "js": - return self.eval_js(script) - elif ENGINE == "rhino": - return self.eval_rhino(script) - else: - results = [] - if PYV8: - res = self.eval_pyv8(script) - print "PyV8:", res - results.append(res) - if JS: - res = self.eval_js(script) - print "JS:", res - results.append(res) - if RHINO: - res = self.eval_rhino(script) - print "Rhino:", res - results.append(res) - - warning = False - for x in results: - for y in results: - if x != y: - warning = True - - if warning: print "### WARNING ###: Different results" - - return results[0] - - def eval_pyv8(self, script): - rt = PyV8.JSContext() - rt.enter() - return rt.eval(script) - - def eval_js(self, script): - script = "print(eval(unescape('%s')))" % quote(script) - p = subprocess.Popen(["js", "-e", script], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) - out, err = p.communicate() - res = out.strip() - return res - - def eval_rhino(self, script): - script = "print(eval(unescape('%s')))" % quote(script) - p = subprocess.Popen(["java", "-cp", path, "org.mozilla.javascript.tools.shell.Main", "-e", script], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) - out, err = p.communicate() - res = out.strip() - return res.decode("utf8").encode("ISO-8859-1") - - def error(self): - return _("No js engine detected, please install either Spidermonkey, ossp-js, pyv8 or rhino") - -if __name__ == "__main__": - js = JsEngine() - - test = u'"ü"+"ä"' - js.eval(test)
\ No newline at end of file diff --git a/module/common/__init__.py b/module/common/__init__.py deleted file mode 100644 index de6d13128..000000000 --- a/module/common/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -__author__ = 'christian' -
\ No newline at end of file diff --git a/module/common/json_layer.py b/module/common/json_layer.py deleted file mode 100644 index 4d57a9f38..000000000 --- a/module/common/json_layer.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -# abstraction layer for json operations - -try: # since python 2.6 - import json - from json import loads as json_loads - from json import dumps as json_dumps -except ImportError: #use system simplejson if available - import simplejson as json - from simplejson import loads as json_loads - from simplejson import dumps as json_dumps diff --git a/module/common/packagetools.py b/module/common/packagetools.py deleted file mode 100644 index d930157e1..000000000 --- a/module/common/packagetools.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- - -import re - -from urlparse import urlparse - - -endings = ("jdeatme", "3gp", "7zip", "7z", "abr", "ac3", "aiff", "aifc", "aif", "ai", - "au", "avi", "apk", "bin", "bmp", "bat", "bz2", "cbr", "cbz", "ccf", "chm", - "cr2", "cso", "cue", "cvd", "dta", "deb", "divx", "djvu", "dlc", "dmg", "doc", - "docx", "dot", "eps", "epub", "exe", "ff", "flv", "flac", "f4v", "gsd", "gif", - "gpg", "gz", "iwd", "idx", "iso", "ipa", "ipsw", "java", "jar", "jpe?g", "load", - "m2ts", "m4v", "m4a", "md5", "mkv", "mp2", "mp3", "mp4", "mobi", "mov", "movie", - "mpeg", "mpe", "mpg", "mpq", "msi", "msu", "msp", "mv", "mws", "nfo", "npk", "oga", - "ogg", "ogv", "otrkey", "par2", "pkg", "png", "pdf", "pptx?", "ppsx?", "ppz", "pot", - "psd", "qt", "rmvb", "rm", "rar", "ram", "ra", "rev", "rnd", "rpm", "run", "rsdf", - "reg", "rtf", "shnf", "sh(?!tml)", "ssa", "smi", "sub", "srt", "snd", "sfv", "sfx", - "swf", "swc", "tar\.(gz|bz2|xz)", "tar", "tgz", "tiff?", "ts", "txt", "viv", "vivo", - "vob", "vtt", "webm", "wav", "wmv", "wma", "xla", "xls", "xpi", "zeno", "zip", - "[r-z]\d{2}", "_[_a-z]{2}", "\d{3,4}(?=\?|$|\"|\r|\n)") - -rarPats = [re.compile(r'(.*)(\.|_|-)pa?r?t?\.?\d+.(rar|exe)$', re.I), - re.compile(r'(.*)(\.|_|-)part\.?[0]*[1].(rar|exe)$', re.I), - re.compile(r'(.*)\.rar$', re.I), - re.compile(r'(.*)\.r\d+$', re.I), - re.compile(r'(.*)(\.|_|-)\d+$', re.I)] - -zipPats = [re.compile(r'(.*)\.zip$', re.I), - re.compile(r'(.*)\.z\d+$', re.I), - re.compile(r'(?is).*\.7z\.[\d]+$', re.I), - re.compile(r'(.*)\.a.$', re.I)] - -ffsjPats = [re.compile(r'(.*)\._((_[a-z])|([a-z]{2}))(\.|$)'), - re.compile(r'(.*)(\.|_|-)[\d]+(\.(' + '|'.join(endings) + ')$)', re.I)] - -iszPats = [re.compile(r'(.*)\.isz$', re.I), - re.compile(r'(.*)\.i\d{2}$', re.I)] - -pat0 = re.compile(r'www\d*\.', re.I) - -pat1 = re.compile(r'(\.?CD\d+)', re.I) -pat2 = re.compile(r'(\.?part\d+)', re.I) - -pat3 = re.compile(r'(.+)[\.\-_]+$') -pat4 = re.compile(r'(.+)\.\d+\.xtm$') - - -def matchFirst(string, *args): - """ matches against list of regexp and returns first match """ - for patternlist in args: - for pattern in patternlist: - m = pattern.search(string) - if m is not None: - name = m.group(1) - return name - - return string - - -def parseNames(files): - """ Generates packages names from name, data lists - - :param files: list of (name, data) - :return: packagenames mapped to data lists (eg. urls) - """ - packs = {} - - for file, url in files: - patternMatch = False - - if file is None: - continue - - # remove trailing / - name = file.rstrip('/') - - # extract last path part .. if there is a path - split = name.rsplit("/", 1) - if len(split) > 1: - name = split.pop(1) - - #check if an already existing package may be ok for this file - # found = False - # for pack in packs: - # if pack in file: - # packs[pack].append(url) - # found = True - # break - # - # if found: - # continue - - # unrar pattern, 7zip/zip and hjmerge pattern, isz pattern, FFSJ pattern - before = name - name = matchFirst(name, rarPats, zipPats, iszPats, ffsjPats) - if before != name: - patternMatch = True - - # xtremsplit pattern - m = pat4.search(name) - if m is not None: - name = m.group(1) - - # remove part and cd pattern - m = pat1.search(name) - if m is not None: - name = name.replace(m.group(0), "") - patternMatch = True - - m = pat2.search(name) - if m is not None: - name = name.replace(m.group(0), "") - patternMatch = True - - # additional checks if extension pattern matched - if patternMatch: - # remove extension - index = name.rfind(".") - if index <= 0: - index = name.rfind("_") - if index > 0: - length = len(name) - index - if length <= 4: - name = name[:-length] - - # remove endings like . _ - - m = pat3.search(name) - if m is not None: - name = m.group(1) - - # replace . and _ with space - name = name.replace(".", " ") - name = name.replace("_", " ") - - name = name.strip() - else: - name = "" - - # fallback: package by hoster - if not name: - name = urlparse(file).hostname - if name: - name = pat0.sub("", name) - - # fallback : default name - if not name: - name = _("Unnamed package") - - # build mapping - if name in packs: - packs[name].append(url) - else: - packs[name] = [url] - - return packs diff --git a/module/common/pylgettext.py b/module/common/pylgettext.py deleted file mode 100644 index fb36fecee..000000000 --- a/module/common/pylgettext.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from gettext import * - -_searchdirs = None - -origfind = find - -def setpaths(pathlist): - global _searchdirs - if isinstance(pathlist, list): - _searchdirs = pathlist - else: - _searchdirs = list(pathlist) - - -def addpath(path): - global _searchdirs - if _searchdirs is None: - _searchdirs = list(path) - else: - if path not in _searchdirs: - _searchdirs.append(path) - - -def delpath(path): - global _searchdirs - if _searchdirs is not None: - if path in _searchdirs: - _searchdirs.remove(path) - - -def clearpath(): - global _searchdirs - if _searchdirs is not None: - _searchdirs = None - - -def find(domain, localedir=None, languages=None, all=False): - if _searchdirs is None: - return origfind(domain, localedir, languages, all) - searches = [localedir] + _searchdirs - results = list() - for dir in searches: - res = origfind(domain, dir, languages, all) - if all is False: - results.append(res) - else: - results.extend(res) - if all is False: - results = filter(lambda x: x is not None, results) - if len(results) == 0: - return None - else: - return results[0] - else: - return results - -#Is there a smarter/cleaner pythonic way for this? -translation.func_globals['find'] = find |